2024-12-03 01:10:52,979 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-03 01:10:52,992 main DEBUG Took 0.010138 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-03 01:10:52,992 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-03 01:10:52,992 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-03 01:10:52,993 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-03 01:10:52,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,001 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-03 01:10:53,012 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,014 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,015 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,018 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,019 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,021 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,023 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,024 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,026 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,027 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-03 01:10:53,028 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,028 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-03 01:10:53,030 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-03 01:10:53,032 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-03 01:10:53,035 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-03 01:10:53,035 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-03 01:10:53,037 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-03 01:10:53,037 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-03 01:10:53,050 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-03 01:10:53,054 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-03 01:10:53,056 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-03 01:10:53,056 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-03 01:10:53,057 main DEBUG createAppenders(={Console})
2024-12-03 01:10:53,058 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc initialized
2024-12-03 01:10:53,058 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc
2024-12-03 01:10:53,059 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@55de24cc OK.
2024-12-03 01:10:53,059 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-03 01:10:53,060 main DEBUG OutputStream closed
2024-12-03 01:10:53,060 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-03 01:10:53,061 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-03 01:10:53,061 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@53ce1329 OK
2024-12-03 01:10:53,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-03 01:10:53,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-03 01:10:53,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-03 01:10:53,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-03 01:10:53,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-03 01:10:53,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-03 01:10:53,145 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-03 01:10:53,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-03 01:10:53,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-03 01:10:53,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-03 01:10:53,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-03 01:10:53,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-03 01:10:53,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-03 01:10:53,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-03 01:10:53,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-03 01:10:53,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-03 01:10:53,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-03 01:10:53,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-03 01:10:53,152 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-03 01:10:53,153 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-03 01:10:53,153 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-03 01:10:53,154 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-03T01:10:53,475 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6
2024-12-03 01:10:53,479 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-03 01:10:53,480 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-03T01:10:53,493 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-03T01:10:53,539 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=286, MaxFileDescriptor=1048576, SystemLoadAverage=853, ProcessCount=11, AvailableMemoryMB=3894
2024-12-03T01:10:53,543 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:10:53,546 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3, deleteOnExit=true
2024-12-03T01:10:53,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:10:53,548 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/test.cache.data in system properties and HBase conf
2024-12-03T01:10:53,548 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:10:53,549 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:10:53,550 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:10:53,550 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:10:53,551 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:10:53,711 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-03T01:10:53,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:10:53,877 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:10:53,878 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:10:53,879 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:10:53,879 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:10:53,880 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:10:53,880 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:10:53,881 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:10:53,882 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:10:53,882 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:10:53,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:10:53,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:10:53,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:10:53,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:10:53,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:10:54,496 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:10:54,881 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-03T01:10:54,981 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2980ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-03T01:10:55,074 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:10:55,175 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:10:55,219 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:10:55,219 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:10:55,221 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:10:55,242 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:10:55,246 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:10:55,247 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:10:55,496 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5682c4d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/java.io.tmpdir/jetty-localhost-45031-hadoop-hdfs-3_4_1-tests_jar-_-any-3147781903749557493/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:10:55,503 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:45031}
2024-12-03T01:10:55,504 INFO  [Time-limited test {}] server.Server(415): Started @3503ms
2024-12-03T01:10:55,530 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:10:55,914 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:10:55,922 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:10:55,923 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:10:55,924 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:10:55,924 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:10:55,925 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:10:55,925 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:10:56,048 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6aad8790{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/java.io.tmpdir/jetty-localhost-38953-hadoop-hdfs-3_4_1-tests_jar-_-any-1300027110738342140/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:10:56,049 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:38953}
2024-12-03T01:10:56,049 INFO  [Time-limited test {}] server.Server(415): Started @4048ms
2024-12-03T01:10:56,107 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:10:56,266 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:10:56,273 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:10:56,281 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:10:56,281 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:10:56,281 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:10:56,285 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:10:56,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:10:56,431 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@163cfad6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/java.io.tmpdir/jetty-localhost-44379-hadoop-hdfs-3_4_1-tests_jar-_-any-5278081928813372821/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:10:56,432 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:44379}
2024-12-03T01:10:56,432 INFO  [Time-limited test {}] server.Server(415): Started @4431ms
2024-12-03T01:10:56,434 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:10:56,633 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data4/current/BP-218709567-172.17.0.2-1733188254599/current, will proceed with Du for space computation calculation, 
2024-12-03T01:10:56,633 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data1/current/BP-218709567-172.17.0.2-1733188254599/current, will proceed with Du for space computation calculation, 
2024-12-03T01:10:56,633 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data3/current/BP-218709567-172.17.0.2-1733188254599/current, will proceed with Du for space computation calculation, 
2024-12-03T01:10:56,633 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data2/current/BP-218709567-172.17.0.2-1733188254599/current, will proceed with Du for space computation calculation, 
2024-12-03T01:10:56,702 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:10:56,703 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:10:56,772 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f5284303b8219ff with lease ID 0xb3fb1c8c0289ac3f: Processing first storage report for DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30 from datanode DatanodeRegistration(127.0.0.1:42289, datanodeUuid=b736c9d1-7911-41c1-9f05-0b8863b05fee, infoPort=46765, infoSecurePort=0, ipcPort=40959, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599)
2024-12-03T01:10:56,773 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f5284303b8219ff with lease ID 0xb3fb1c8c0289ac3f: from storage DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30 node DatanodeRegistration(127.0.0.1:42289, datanodeUuid=b736c9d1-7911-41c1-9f05-0b8863b05fee, infoPort=46765, infoSecurePort=0, ipcPort=40959, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:10:56,773 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4c772dcfed30b3c with lease ID 0xb3fb1c8c0289ac3e: Processing first storage report for DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a from datanode DatanodeRegistration(127.0.0.1:43133, datanodeUuid=33a7291a-d48a-407d-bf7b-0c0429d7dbb6, infoPort=36535, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599)
2024-12-03T01:10:56,774 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4c772dcfed30b3c with lease ID 0xb3fb1c8c0289ac3e: from storage DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a node DatanodeRegistration(127.0.0.1:43133, datanodeUuid=33a7291a-d48a-407d-bf7b-0c0429d7dbb6, infoPort=36535, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:10:56,774 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f5284303b8219ff with lease ID 0xb3fb1c8c0289ac3f: Processing first storage report for DS-b1e1fc69-140d-4aee-b4f5-cc345727fc0f from datanode DatanodeRegistration(127.0.0.1:42289, datanodeUuid=b736c9d1-7911-41c1-9f05-0b8863b05fee, infoPort=46765, infoSecurePort=0, ipcPort=40959, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599)
2024-12-03T01:10:56,774 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f5284303b8219ff with lease ID 0xb3fb1c8c0289ac3f: from storage DS-b1e1fc69-140d-4aee-b4f5-cc345727fc0f node DatanodeRegistration(127.0.0.1:42289, datanodeUuid=b736c9d1-7911-41c1-9f05-0b8863b05fee, infoPort=46765, infoSecurePort=0, ipcPort=40959, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:10:56,774 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4c772dcfed30b3c with lease ID 0xb3fb1c8c0289ac3e: Processing first storage report for DS-b323dc28-228b-4257-9363-f1b49da668c5 from datanode DatanodeRegistration(127.0.0.1:43133, datanodeUuid=33a7291a-d48a-407d-bf7b-0c0429d7dbb6, infoPort=36535, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599)
2024-12-03T01:10:56,774 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4c772dcfed30b3c with lease ID 0xb3fb1c8c0289ac3e: from storage DS-b323dc28-228b-4257-9363-f1b49da668c5 node DatanodeRegistration(127.0.0.1:43133, datanodeUuid=33a7291a-d48a-407d-bf7b-0c0429d7dbb6, infoPort=36535, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1115938548;c=1733188254599), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:10:56,863 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6
2024-12-03T01:10:56,944 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/zookeeper_0, clientPort=49500, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:10:56,955 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=49500
2024-12-03T01:10:56,966 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:56,969 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:57,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:10:57,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:10:57,669 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329 with version=8
2024-12-03T01:10:57,670 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:10:57,819 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-03T01:10:58,096 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:10:58,117 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,117 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,117 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:10:58,118 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,118 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:10:58,263 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:10:58,335 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-03T01:10:58,344 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-03T01:10:58,349 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:10:58,377 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28706 (auto-detected)
2024-12-03T01:10:58,378 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-03T01:10:58,396 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38995
2024-12-03T01:10:58,404 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:58,407 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:58,422 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38995 connecting to ZooKeeper ensemble=127.0.0.1:49500
2024-12-03T01:10:58,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389950x0, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:10:58,464 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38995-0x10175dfef2e0000 connected
2024-12-03T01:10:58,494 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:10:58,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:10:58,500 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:10:58,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38995
2024-12-03T01:10:58,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38995
2024-12-03T01:10:58,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38995
2024-12-03T01:10:58,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38995
2024-12-03T01:10:58,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38995
2024-12-03T01:10:58,512 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329, hbase.cluster.distributed=false
2024-12-03T01:10:58,577 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:10:58,577 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,577 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,577 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:10:58,578 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:10:58,578 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:10:58,580 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:10:58,583 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:10:58,584 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41559
2024-12-03T01:10:58,586 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:10:58,593 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:10:58,594 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:58,599 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:58,603 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:41559 connecting to ZooKeeper ensemble=127.0.0.1:49500
2024-12-03T01:10:58,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415590x0, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:10:58,612 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:415590x0, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:10:58,612 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41559-0x10175dfef2e0001 connected
2024-12-03T01:10:58,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:10:58,615 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:10:58,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41559
2024-12-03T01:10:58,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41559
2024-12-03T01:10:58,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41559
2024-12-03T01:10:58,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41559
2024-12-03T01:10:58,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41559
2024-12-03T01:10:58,627 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:58,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:10:58,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:10:58,636 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:58,644 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:38995
2024-12-03T01:10:58,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:10:58,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:10:58,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:58,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:58,660 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:10:58,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:10:58,661 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,38995,1733188257813 from backup master directory
2024-12-03T01:10:58,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:58,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:10:58,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:10:58,670 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:10:58,670 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:58,673 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-03T01:10:58,675 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-03T01:10:58,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:10:58,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:10:58,782 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase.id with ID: a40bc929-73df-4491-a5ba-812af823e53f
2024-12-03T01:10:58,845 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:10:58,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:58,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:58,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:10:58,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:10:58,907 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:10:58,909 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:10:58,915 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:10:58,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:10:58,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:10:58,975 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store
2024-12-03T01:10:58,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:10:58,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:10:59,002 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-03T01:10:59,002 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:10:59,003 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:10:59,003 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:10:59,003 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:10:59,003 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:10:59,004 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:10:59,004 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:10:59,004 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:10:59,006 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/.initializing
2024-12-03T01:10:59,006 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/WALs/1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:59,023 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C38995%2C1733188257813, suffix=, logDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/WALs/1a7b1a4db65e,38995,1733188257813, archiveDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/oldWALs, maxLogs=10
2024-12-03T01:10:59,034 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C38995%2C1733188257813.1733188259030
2024-12-03T01:10:59,035 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(752): Using builder API via reflection for DFS file creation replicate flag.
2024-12-03T01:10:59,035 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.CommonFSUtils$DfsBuilderUtility(762): Using builder API via reflection for DFS file creation noLocalWrite flag.
2024-12-03T01:10:59,058 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/WALs/1a7b1a4db65e,38995,1733188257813/1a7b1a4db65e%2C38995%2C1733188257813.1733188259030
2024-12-03T01:10:59,070 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46765:46765),(127.0.0.1/127.0.0.1:36535:36535)]
2024-12-03T01:10:59,070 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:10:59,071 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:10:59,075 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:10:59,077 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:10:59,126 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:10:59,162 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:10:59,167 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,170 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:10:59,170 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:10:59,174 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:10:59,174 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,175 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:10:59,175 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:10:59,178 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:10:59,178 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:10:59,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:10:59,182 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:10:59,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,183 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:10:59,187 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:10:59,188 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:10:59,198 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:10:59,202 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:10:59,207 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:10:59,208 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=721794, jitterRate=-0.08219222724437714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:10:59,212 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:10:59,213 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:10:59,244 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467b1b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:10:59,282 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:10:59,293 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:10:59,294 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:10:59,296 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:10:59,297 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec
2024-12-03T01:10:59,302 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec
2024-12-03T01:10:59,302 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:10:59,338 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:10:59,354 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:10:59,356 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:10:59,359 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:10:59,360 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:10:59,361 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:10:59,363 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:10:59,366 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:10:59,370 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:10:59,371 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:10:59,372 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:10:59,381 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:10:59,383 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:10:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:10:59,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:10:59,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,387 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,38995,1733188257813, sessionid=0x10175dfef2e0000, setting cluster-up flag (Was=false)
2024-12-03T01:10:59,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,408 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:10:59,410 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:59,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:10:59,421 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:10:59,423 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38995,1733188257813
2024-12-03T01:10:59,502 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:10:59,509 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:10:59,512 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:10:59,518 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,38995,1733188257813
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:10:59,521 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:10:59,521 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:10:59,522 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,525 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188289524
2024-12-03T01:10:59,526 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:10:59,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:10:59,527 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:10:59,527 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:10:59,530 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:10:59,530 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:10:59,530 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:10:59,531 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:10:59,531 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,531 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,532 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:10:59,532 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:10:59,533 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:10:59,537 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:10:59,539 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:10:59,539 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:10:59,542 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188259541,5,FailOnTimeoutGroup]
2024-12-03T01:10:59,542 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188259542,5,FailOnTimeoutGroup]
2024-12-03T01:10:59,542 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,543 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:10:59,543 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:41559
2024-12-03T01:10:59,544 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,544 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,545 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1008): ClusterId : a40bc929-73df-4491-a5ba-812af823e53f
2024-12-03T01:10:59,548 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:10:59,548 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:10:59,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:10:59,554 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:10:59,555 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329
2024-12-03T01:10:59,557 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:10:59,557 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:10:59,560 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:10:59,561 DEBUG [RS:0;1a7b1a4db65e:41559 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703f71ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:10:59,563 DEBUG [RS:0;1a7b1a4db65e:41559 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af4123c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:10:59,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:10:59,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:10:59,567 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:10:59,568 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:10:59,568 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:10:59,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:10:59,571 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,38995,1733188257813 with isa=1a7b1a4db65e/172.17.0.2:41559, startcode=1733188258576
2024-12-03T01:10:59,576 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:10:59,579 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:10:59,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:10:59,581 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:10:59,583 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:10:59,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,584 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:10:59,584 DEBUG [RS:0;1a7b1a4db65e:41559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:10:59,584 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:10:59,587 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:10:59,587 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:10:59,588 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:10:59,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740
2024-12-03T01:10:59,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740
2024-12-03T01:10:59,594 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:10:59,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:10:59,602 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:10:59,603 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=711769, jitterRate=-0.0949401706457138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:10:59,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:10:59,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:10:59,606 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:10:59,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:10:59,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:10:59,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:10:59,607 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:10:59,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:10:59,611 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:10:59,611 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:10:59,618 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:10:59,629 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:10:59,633 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:10:59,669 INFO  [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36051, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:10:59,676 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38995 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,679 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38995 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,697 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329
2024-12-03T01:10:59,698 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:45713
2024-12-03T01:10:59,698 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:10:59,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:10:59,703 DEBUG [RS:0;1a7b1a4db65e:41559 {}] zookeeper.ZKUtil(111): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,703 WARN  [RS:0;1a7b1a4db65e:41559 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:10:59,703 INFO  [RS:0;1a7b1a4db65e:41559 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:10:59,703 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,705 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,41559,1733188258576]
2024-12-03T01:10:59,717 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:10:59,729 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:10:59,740 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:10:59,743 INFO  [RS:0;1a7b1a4db65e:41559 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:10:59,744 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,744 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:10:59,751 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,751 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,751 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,752 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,753 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:10:59,753 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:10:59,753 DEBUG [RS:0;1a7b1a4db65e:41559 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:10:59,754 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,754 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,755 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,756 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,756 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,41559,1733188258576-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:10:59,784 WARN  [1a7b1a4db65e:38995 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-03T01:10:59,787 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:10:59,789 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,41559,1733188258576-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:10:59,815 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.Replication(204): 1a7b1a4db65e,41559,1733188258576 started
2024-12-03T01:10:59,816 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,41559,1733188258576, RpcServer on 1a7b1a4db65e/172.17.0.2:41559, sessionid=0x10175dfef2e0001
2024-12-03T01:10:59,816 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:10:59,817 DEBUG [RS:0;1a7b1a4db65e:41559 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,817 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,41559,1733188258576'
2024-12-03T01:10:59,817 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:10:59,818 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:10:59,819 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:10:59,819 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:10:59,820 DEBUG [RS:0;1a7b1a4db65e:41559 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:10:59,820 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,41559,1733188258576'
2024-12-03T01:10:59,820 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:10:59,820 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:10:59,821 DEBUG [RS:0;1a7b1a4db65e:41559 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:10:59,822 INFO  [RS:0;1a7b1a4db65e:41559 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:10:59,822 INFO  [RS:0;1a7b1a4db65e:41559 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:10:59,931 INFO  [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C41559%2C1733188258576, suffix=, logDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576, archiveDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs, maxLogs=32
2024-12-03T01:10:59,934 INFO  [RS:0;1a7b1a4db65e:41559 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188259934
2024-12-03T01:10:59,945 INFO  [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188259934
2024-12-03T01:10:59,945 DEBUG [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46765:46765),(127.0.0.1/127.0.0.1:36535:36535)]
2024-12-03T01:11:00,036 DEBUG [1a7b1a4db65e:38995 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:11:00,040 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,045 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,41559,1733188258576, state=OPENING
2024-12-03T01:11:00,051 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:11:00,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:11:00,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:11:00,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:11:00,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:11:00,056 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,41559,1733188258576}]
2024-12-03T01:11:00,232 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,234 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:11:00,237 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:11:00,248 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:11:00,249 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:11:00,252 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C41559%2C1733188258576.meta, suffix=.meta, logDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576, archiveDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs, maxLogs=32
2024-12-03T01:11:00,254 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.meta.1733188260254.meta
2024-12-03T01:11:00,263 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.meta.1733188260254.meta
2024-12-03T01:11:00,263 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:11:00,263 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:11:00,265 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:11:00,330 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:11:00,336 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:11:00,342 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:11:00,342 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:11:00,342 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:11:00,342 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:11:00,347 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:11:00,350 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:11:00,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:00,351 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:11:00,351 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:11:00,353 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:11:00,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:00,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:11:00,354 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:11:00,356 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:11:00,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:00,357 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:11:00,359 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740
2024-12-03T01:11:00,362 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740
2024-12-03T01:11:00,365 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:11:00,368 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:11:00,370 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703845, jitterRate=-0.10501515865325928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:11:00,372 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:11:00,379 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188260227
2024-12-03T01:11:00,394 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:11:00,395 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:11:00,396 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,398 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,41559,1733188258576, state=OPEN
2024-12-03T01:11:00,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:11:00,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:11:00,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:11:00,405 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:11:00,410 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:11:00,410 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,41559,1733188258576 in 349 msec
2024-12-03T01:11:00,416 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:11:00,416 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 794 msec
2024-12-03T01:11:00,421 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 963 msec
2024-12-03T01:11:00,422 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188260422, completionTime=-1
2024-12-03T01:11:00,422 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:11:00,422 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:11:00,459 DEBUG [hconnection-0x5f3d2985-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:11:00,461 INFO  [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44412, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:11:00,473 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:11:00,473 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188320473
2024-12-03T01:11:00,473 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188380473
2024-12-03T01:11:00,474 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 51 msec
2024-12-03T01:11:00,502 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:00,502 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:00,502 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:00,504 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:38995, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:00,504 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:00,511 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:11:00,513 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:11:00,514 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:11:00,521 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:11:00,524 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:11:00,525 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:00,527 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:11:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:11:00,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:11:00,542 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => acc8878667541fe51516b373c2b39ae9, NAME => 'hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329
2024-12-03T01:11:00,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:11:00,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:11:00,552 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:11:00,552 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing acc8878667541fe51516b373c2b39ae9, disabling compactions & flushes
2024-12-03T01:11:00,552 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,552 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,552 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9. after waiting 0 ms
2024-12-03T01:11:00,552 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,552 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for acc8878667541fe51516b373c2b39ae9:

2024-12-03T01:11:00,555 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:11:00,561 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188260556"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188260556"}]},"ts":"1733188260556"}
2024-12-03T01:11:00,585 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:11:00,587 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:11:00,590 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188260587"}]},"ts":"1733188260587"}
2024-12-03T01:11:00,595 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:11:00,602 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=acc8878667541fe51516b373c2b39ae9, ASSIGN}]
2024-12-03T01:11:00,604 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=acc8878667541fe51516b373c2b39ae9, ASSIGN
2024-12-03T01:11:00,606 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=acc8878667541fe51516b373c2b39ae9, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,41559,1733188258576; forceNewPlan=false, retain=false
2024-12-03T01:11:00,757 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=acc8878667541fe51516b373c2b39ae9, regionState=OPENING, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,761 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure acc8878667541fe51516b373c2b39ae9, server=1a7b1a4db65e,41559,1733188258576}]
2024-12-03T01:11:00,915 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,924 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => acc8878667541fe51516b373c2b39ae9, NAME => 'hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:11:00,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:11:00,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,928 INFO  [StoreOpener-acc8878667541fe51516b373c2b39ae9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region acc8878667541fe51516b373c2b39ae9 
2024-12-03T01:11:00,930 INFO  [StoreOpener-acc8878667541fe51516b373c2b39ae9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acc8878667541fe51516b373c2b39ae9 columnFamilyName info
2024-12-03T01:11:00,930 DEBUG [StoreOpener-acc8878667541fe51516b373c2b39ae9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:00,931 INFO  [StoreOpener-acc8878667541fe51516b373c2b39ae9-1 {}] regionserver.HStore(327): Store=acc8878667541fe51516b373c2b39ae9/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:11:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,933 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,937 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for acc8878667541fe51516b373c2b39ae9
2024-12-03T01:11:00,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:11:00,942 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened acc8878667541fe51516b373c2b39ae9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=823655, jitterRate=0.04733167588710785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:11:00,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for acc8878667541fe51516b373c2b39ae9:

2024-12-03T01:11:00,946 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9., pid=6, masterSystemTime=1733188260915
2024-12-03T01:11:00,950 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,950 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:11:00,951 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=acc8878667541fe51516b373c2b39ae9, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:00,958 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:11:00,959 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure acc8878667541fe51516b373c2b39ae9, server=1a7b1a4db65e,41559,1733188258576 in 194 msec
2024-12-03T01:11:00,962 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:11:00,962 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=acc8878667541fe51516b373c2b39ae9, ASSIGN in 357 msec
2024-12-03T01:11:00,963 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:11:00,964 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188260963"}]},"ts":"1733188260963"}
2024-12-03T01:11:00,966 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:11:00,971 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:11:00,973 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 456 msec
2024-12-03T01:11:01,024 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:11:01,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:11:01,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:11:01,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:11:01,060 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:11:01,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:11:01,082 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 26 msec
2024-12-03T01:11:01,094 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:11:01,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:11:01,112 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec
2024-12-03T01:11:01,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:11:01,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:11:01,128 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 2.457sec
2024-12-03T01:11:01,130 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:11:01,132 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:11:01,133 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:11:01,134 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:11:01,134 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:11:01,135 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:11:01,136 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:11:01,146 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c4612d8 to 127.0.0.1:49500 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@87fdff8
2024-12-03T01:11:01,147 WARN  [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry
2024-12-03T01:11:01,151 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:11:01,152 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:11:01,153 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38995,1733188257813-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:11:01,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0c09d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:11:01,156 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-03T01:11:01,156 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-03T01:11:01,164 DEBUG [hconnection-0x3878e8d1-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:11:01,172 INFO  [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47434, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:11:01,214 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,38995,1733188257813
2024-12-03T01:11:01,215 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:11:01,226 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:11:01,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T01:11:01,291 INFO  [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50248, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T01:11:01,298 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T01:11:01,298 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T01:11:01,302 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:11:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-03T01:11:01,306 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:11:01,307 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:01,308 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:11:01,309 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 9
2024-12-03T01:11:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:11:01,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741837_1013 (size=389)
2024-12-03T01:11:01,320 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741837_1013 (size=389)
2024-12-03T01:11:01,322 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 3d508b286c6b64ce9e95f4f9bd1763f0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329
2024-12-03T01:11:01,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741838_1014 (size=72)
2024-12-03T01:11:01,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741838_1014 (size=72)
2024-12-03T01:11:01,333 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:11:01,333 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 3d508b286c6b64ce9e95f4f9bd1763f0, disabling compactions & flushes
2024-12-03T01:11:01,333 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,333 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,333 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0. after waiting 0 ms
2024-12-03T01:11:01,334 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,334 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,334 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:11:01,336 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:11:01,336 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733188261336"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188261336"}]},"ts":"1733188261336"}
2024-12-03T01:11:01,339 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:11:01,341 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:11:01,341 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188261341"}]},"ts":"1733188261341"}
2024-12-03T01:11:01,344 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-03T01:11:01,349 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3d508b286c6b64ce9e95f4f9bd1763f0, ASSIGN}]
2024-12-03T01:11:01,351 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3d508b286c6b64ce9e95f4f9bd1763f0, ASSIGN
2024-12-03T01:11:01,353 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3d508b286c6b64ce9e95f4f9bd1763f0, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,41559,1733188258576; forceNewPlan=false, retain=false
2024-12-03T01:11:01,504 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=3d508b286c6b64ce9e95f4f9bd1763f0, regionState=OPENING, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:01,508 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 3d508b286c6b64ce9e95f4f9bd1763f0, server=1a7b1a4db65e,41559,1733188258576}]
2024-12-03T01:11:01,662 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:01,668 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,669 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 3d508b286c6b64ce9e95f4f9bd1763f0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:11:01,669 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,670 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:11:01,670 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,670 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,672 INFO  [StoreOpener-3d508b286c6b64ce9e95f4f9bd1763f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3d508b286c6b64ce9e95f4f9bd1763f0 
2024-12-03T01:11:01,674 INFO  [StoreOpener-3d508b286c6b64ce9e95f4f9bd1763f0-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d508b286c6b64ce9e95f4f9bd1763f0 columnFamilyName info
2024-12-03T01:11:01,675 DEBUG [StoreOpener-3d508b286c6b64ce9e95f4f9bd1763f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:11:01,676 INFO  [StoreOpener-3d508b286c6b64ce9e95f4f9bd1763f0-1 {}] regionserver.HStore(327): Store=3d508b286c6b64ce9e95f4f9bd1763f0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:11:01,677 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,678 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,682 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:01,686 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:11:01,687 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 3d508b286c6b64ce9e95f4f9bd1763f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881271, jitterRate=0.1205943375825882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:11:01,688 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:11:01,689 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0., pid=11, masterSystemTime=1733188261661
2024-12-03T01:11:01,693 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,693 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:01,694 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=3d508b286c6b64ce9e95f4f9bd1763f0, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,41559,1733188258576
2024-12-03T01:11:01,701 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-03T01:11:01,702 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 3d508b286c6b64ce9e95f4f9bd1763f0, server=1a7b1a4db65e,41559,1733188258576 in 189 msec
2024-12-03T01:11:01,704 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T01:11:01,704 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=3d508b286c6b64ce9e95f4f9bd1763f0, ASSIGN in 352 msec
2024-12-03T01:11:01,706 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:11:01,706 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188261706"}]},"ts":"1733188261706"}
2024-12-03T01:11:01,709 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-03T01:11:01,713 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:11:01,716 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 411 msec
2024-12-03T01:11:05,847 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-03T01:11:05,905 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T01:11:05,907 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-03T01:11:05,908 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-03T01:11:08,332 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-03T01:11:08,332 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-03T01:11:08,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-03T01:11:08,335 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T01:11:08,337 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-12-03T01:11:08,337 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer
2024-12-03T01:11:08,338 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T01:11:08,338 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-03T01:11:08,339 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-03T01:11:08,339 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-03T01:11:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38995 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:11:11,322 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling, procId: 9 completed
2024-12-03T01:11:11,326 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-03T01:11:11,327 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:11:11,327 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188271327
2024-12-03T01:11:11,338 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188259934 with entries=4, filesize=947 B; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188271327
2024-12-03T01:11:11,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:11:11,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188259934 is not closed yet, will try archiving it next time
2024-12-03T01:11:11,345 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741833_1009 (size=955)
2024-12-03T01:11:11,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741833_1009 (size=955)
2024-12-03T01:11:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41559 {}] regionserver.HRegion(8581): Flush requested on 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:23,371 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 3d508b286c6b64ce9e95f4f9bd1763f0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:11:23,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/457d846862fb4de58792cf8f30a877f9 is 1080, key is row0001/info:/1733188271345/Put/seqid=0
2024-12-03T01:11:23,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741840_1016 (size=12509)
2024-12-03T01:11:23,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741840_1016 (size=12509)
2024-12-03T01:11:23,455 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/457d846862fb4de58792cf8f30a877f9
2024-12-03T01:11:23,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/457d846862fb4de58792cf8f30a877f9 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9
2024-12-03T01:11:23,515 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9, entries=7, sequenceid=11, filesize=12.2 K
2024-12-03T01:11:23,519 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3d508b286c6b64ce9e95f4f9bd1763f0 in 148ms, sequenceid=11, compaction requested=false
2024-12-03T01:11:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:11:26,860 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:11:29,565 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:11:29,567 INFO  [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40928, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:11:31,382 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188291382
2024-12-03T01:11:31,594 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:31,596 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188271327 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188291382
2024-12-03T01:11:31,596 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:11:31,596 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188271327 is not closed yet, will try archiving it next time
2024-12-03T01:11:31,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741839_1015 (size=12399)
2024-12-03T01:11:31,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741839_1015 (size=12399)
2024-12-03T01:11:31,800 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:34,003 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:36,206 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:38,410 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41559 {}] regionserver.HRegion(8581): Flush requested on 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:11:38,411 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 3d508b286c6b64ce9e95f4f9bd1763f0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:11:38,612 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:38,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/1a671f3bc7d044778977f4a663cae88e is 1080, key is row0008/info:/1733188285372/Put/seqid=0
2024-12-03T01:11:38,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741842_1018 (size=12509)
2024-12-03T01:11:38,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741842_1018 (size=12509)
2024-12-03T01:11:38,627 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/1a671f3bc7d044778977f4a663cae88e
2024-12-03T01:11:38,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/1a671f3bc7d044778977f4a663cae88e as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e
2024-12-03T01:11:38,648 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e, entries=7, sequenceid=21, filesize=12.2 K
2024-12-03T01:11:38,850 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:38,850 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3d508b286c6b64ce9e95f4f9bd1763f0 in 439ms, sequenceid=21, compaction requested=false
2024-12-03T01:11:38,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:11:38,851 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=24.4 K, sizeToCheck=16.0 K
2024-12-03T01:11:38,851 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:11:38,852 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9 because midkey is the same as first or last row
2024-12-03T01:11:40,614 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:41,838 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T01:11:41,838 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T01:11:42,817 WARN  [sync.1 {}] wal.AbstractFSWAL(1346): Requesting log roll because we exceeded slow sync threshold; count=7, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:42,818 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188291382) roll requested
2024-12-03T01:11:42,819 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:42,819 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188302819
2024-12-03T01:11:43,027 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:43,228 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:43,228 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188291382 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188302819
2024-12-03T01:11:43,228 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:11:43,229 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188291382 is not closed yet, will try archiving it next time
2024-12-03T01:11:43,229 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188271327 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188271327
2024-12-03T01:11:43,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741841_1017 (size=7739)
2024-12-03T01:11:43,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741841_1017 (size=7739)
2024-12-03T01:11:45,021 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:46,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3d508b286c6b64ce9e95f4f9bd1763f0, had cached 0 bytes from a total of 25018
2024-12-03T01:11:47,224 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:49,427 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:51,631 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:53,633 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-03T01:11:53,634 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188313633
2024-12-03T01:11:56,861 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:11:58,644 INFO  [Time-limited test {}] wal.AbstractFSWAL(1183): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:58,644 WARN  [Time-limited test {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:11:58,644 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188313633) roll requested
2024-12-03T01:12:00,509 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region acc8878667541fe51516b373c2b39ae9 changed from -1.0 to 0.0, refreshing cache
2024-12-03T01:12:03,644 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:03,645 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:03,646 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188302819 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188313633
2024-12-03T01:12:03,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:12:03,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188302819 is not closed yet, will try archiving it next time
2024-12-03T01:12:03,647 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188323646
2024-12-03T01:12:03,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741843_1019 (size=4753)
2024-12-03T01:12:03,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741843_1019 (size=4753)
2024-12-03T01:12:08,649 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:08,649 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41559 {}] regionserver.HRegion(8581): Flush requested on 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:12:08,649 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 3d508b286c6b64ce9e95f4f9bd1763f0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:12:08,654 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:08,655 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:10,650 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-03T01:12:13,651 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:13,651 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:13,655 INFO  [sync.0 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:13,655 WARN  [sync.0 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:13,655 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188313633 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188323646
2024-12-03T01:12:13,656 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46765:46765),(127.0.0.1/127.0.0.1:36535:36535)]
2024-12-03T01:12:13,656 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188313633 is not closed yet, will try archiving it next time
2024-12-03T01:12:13,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/6c95d1ac48774405b5bb6983368d2688 is 1080, key is row0015/info:/1733188300413/Put/seqid=0
2024-12-03T01:12:13,656 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188323646) roll requested
2024-12-03T01:12:13,656 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188333656
2024-12-03T01:12:13,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741844_1020 (size=1569)
2024-12-03T01:12:13,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741844_1020 (size=1569)
2024-12-03T01:12:13,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741846_1022 (size=12509)
2024-12-03T01:12:13,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741846_1022 (size=12509)
2024-12-03T01:12:13,667 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/6c95d1ac48774405b5bb6983368d2688
2024-12-03T01:12:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/6c95d1ac48774405b5bb6983368d2688 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688
2024-12-03T01:12:13,685 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688, entries=7, sequenceid=31, filesize=12.2 K
2024-12-03T01:12:18,666 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(1183): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:18,666 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:18,686 INFO  [sync.1 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:18,686 WARN  [sync.1 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:18,686 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3d508b286c6b64ce9e95f4f9bd1763f0 in 10037ms, sequenceid=31, compaction requested=true
2024-12-03T01:12:18,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:12:18,687 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=36.6 K, sizeToCheck=16.0 K
2024-12-03T01:12:18,687 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:12:18,687 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9 because midkey is the same as first or last row
2024-12-03T01:12:18,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d508b286c6b64ce9e95f4f9bd1763f0:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:12:18,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:12:18,688 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:12:18,691 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:12:18,693 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HStore(1540): 3d508b286c6b64ce9e95f4f9bd1763f0/info is initiating minor compaction (all files)
2024-12-03T01:12:18,693 INFO  [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 3d508b286c6b64ce9e95f4f9bd1763f0/info in TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:18,693 INFO  [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688] into tmpdir=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp, totalSize=36.6 K
2024-12-03T01:12:18,694 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] compactions.Compactor(224): Compacting 457d846862fb4de58792cf8f30a877f9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733188271345
2024-12-03T01:12:18,695 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1a671f3bc7d044778977f4a663cae88e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733188285372
2024-12-03T01:12:18,696 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6c95d1ac48774405b5bb6983368d2688, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733188300413
2024-12-03T01:12:18,725 INFO  [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d508b286c6b64ce9e95f4f9bd1763f0#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:12:18,727 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/25ec980edc084055ac7842694d15cd07 is 1080, key is row0001/info:/1733188271345/Put/seqid=0
2024-12-03T01:12:18,734 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741848_1024 (size=27710)
2024-12-03T01:12:18,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741848_1024 (size=27710)
2024-12-03T01:12:18,745 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/25ec980edc084055ac7842694d15cd07 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/25ec980edc084055ac7842694d15cd07
2024-12-03T01:12:23,667 INFO  [sync.2 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:23,667 WARN  [sync.2 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK], DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK]]
2024-12-03T01:12:23,668 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188323646 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188333656
2024-12-03T01:12:23,668 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:12:23,669 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188323646 is not closed yet, will try archiving it next time
2024-12-03T01:12:23,669 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188291382 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188291382
2024-12-03T01:12:23,669 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188343669
2024-12-03T01:12:23,670 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188343669) roll requested
2024-12-03T01:12:23,671 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188302819 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188302819
2024-12-03T01:12:23,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741845_1021 (size=438)
2024-12-03T01:12:23,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741845_1021 (size=438)
2024-12-03T01:12:23,673 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188313633 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188313633
2024-12-03T01:12:23,674 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188323646 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188323646
2024-12-03T01:12:26,861 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:12:28,670 INFO  [sync.3 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:28,670 WARN  [sync.3 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:28,672 INFO  [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 3d508b286c6b64ce9e95f4f9bd1763f0/info of 3d508b286c6b64ce9e95f4f9bd1763f0 into 25ec980edc084055ac7842694d15cd07(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 9sec to execute.
2024-12-03T01:12:28,672 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:12:28,672 INFO  [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0., storeName=3d508b286c6b64ce9e95f4f9bd1763f0/info, priority=13, startTime=1733188338688; duration=9sec
2024-12-03T01:12:28,673 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=27.1 K, sizeToCheck=16.0 K
2024-12-03T01:12:28,673 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:12:28,673 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/25ec980edc084055ac7842694d15cd07 because midkey is the same as first or last row
2024-12-03T01:12:28,673 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:12:28,673 DEBUG [RS:0;1a7b1a4db65e:41559-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d508b286c6b64ce9e95f4f9bd1763f0:info
2024-12-03T01:12:28,679 INFO  [sync.4 {}] wal.AbstractFSWAL(1183): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:28,679 WARN  [sync.4 {}] wal.AbstractFSWAL(1189): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43133,DS-bb52ab9a-b65d-4d4d-8101-d9eb5b655e2a,DISK], DatanodeInfoWithStorage[127.0.0.1:42289,DS-cf8b4033-025c-4a2b-9b96-85e11d48cf30,DISK]]
2024-12-03T01:12:28,680 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188333656 with entries=1, filesize=531 B; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188343669
2024-12-03T01:12:28,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:12:28,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188333656 is not closed yet, will try archiving it next time
2024-12-03T01:12:28,680 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188348680
2024-12-03T01:12:28,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741847_1023 (size=539)
2024-12-03T01:12:28,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741847_1023 (size=539)
2024-12-03T01:12:28,685 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188333656 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188333656
2024-12-03T01:12:28,792 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188343669 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188348680
2024-12-03T01:12:28,792 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36535:36535),(127.0.0.1/127.0.0.1:46765:46765)]
2024-12-03T01:12:28,792 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188343669 is not closed yet, will try archiving it next time
2024-12-03T01:12:28,792 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188348680) roll requested
2024-12-03T01:12:28,793 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C41559%2C1733188258576.1733188348792
2024-12-03T01:12:28,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741849_1025 (size=1258)
2024-12-03T01:12:28,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741849_1025 (size=1258)
2024-12-03T01:12:28,888 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188348680 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188348792
2024-12-03T01:12:28,889 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46765:46765),(127.0.0.1/127.0.0.1:36535:36535)]
2024-12-03T01:12:28,889 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188348680 is not closed yet, will try archiving it next time
2024-12-03T01:12:28,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741850_1026 (size=93)
2024-12-03T01:12:28,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741850_1026 (size=93)
2024-12-03T01:12:28,907 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576/1a7b1a4db65e%2C41559%2C1733188258576.1733188348680 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs/1a7b1a4db65e%2C41559%2C1733188258576.1733188348680
2024-12-03T01:12:31,670 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 3d508b286c6b64ce9e95f4f9bd1763f0, had cached 0 bytes from a total of 27710
2024-12-03T01:12:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41559 {}] regionserver.HRegion(8581): Flush requested on 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:12:40,705 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 3d508b286c6b64ce9e95f4f9bd1763f0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:12:40,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/4be31782c4db4640b12ad296d57919cd is 1080, key is row0022/info:/1733188348682/Put/seqid=0
2024-12-03T01:12:40,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741852_1028 (size=12509)
2024-12-03T01:12:40,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741852_1028 (size=12509)
2024-12-03T01:12:40,737 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/4be31782c4db4640b12ad296d57919cd
2024-12-03T01:12:40,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/4be31782c4db4640b12ad296d57919cd as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/4be31782c4db4640b12ad296d57919cd
2024-12-03T01:12:40,768 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/4be31782c4db4640b12ad296d57919cd, entries=7, sequenceid=42, filesize=12.2 K
2024-12-03T01:12:40,770 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 3d508b286c6b64ce9e95f4f9bd1763f0 in 66ms, sequenceid=42, compaction requested=false
2024-12-03T01:12:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:12:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=39.3 K, sizeToCheck=16.0 K
2024-12-03T01:12:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:12:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/25ec980edc084055ac7842694d15cd07 because midkey is the same as first or last row
2024-12-03T01:12:48,714 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:12:48,715 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-03T01:12:48,715 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c4612d8 to 127.0.0.1:49500
2024-12-03T01:12:48,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:12:48,716 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:12:48,716 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=412329742, stopped=false
2024-12-03T01:12:48,716 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,38995,1733188257813
2024-12-03T01:12:48,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:12:48,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:12:48,718 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:12:48,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:48,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:48,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:12:48,718 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,41559,1733188258576' *****
2024-12-03T01:12:48,718 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:12:48,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:12:48,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:12:48,719 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:12:48,719 INFO  [RS:0;1a7b1a4db65e:41559 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:12:48,719 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:12:48,719 INFO  [RS:0;1a7b1a4db65e:41559 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:12:48,719 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(3579): Received CLOSE for acc8878667541fe51516b373c2b39ae9
2024-12-03T01:12:48,720 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(3579): Received CLOSE for 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:12:48,720 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,41559,1733188258576
2024-12-03T01:12:48,720 DEBUG [RS:0;1a7b1a4db65e:41559 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:12:48,720 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:12:48,720 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:12:48,721 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:12:48,721 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing acc8878667541fe51516b373c2b39ae9, disabling compactions & flushes
2024-12-03T01:12:48,721 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:12:48,721 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:12:48,721 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:12:48,721 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9. after waiting 0 ms
2024-12-03T01:12:48,721 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:12:48,721 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing acc8878667541fe51516b373c2b39ae9 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:12:48,721 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-03T01:12:48,721 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1603): Online Regions={acc8878667541fe51516b373c2b39ae9=hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9., 3d508b286c6b64ce9e95f4f9bd1763f0=TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0., 1588230740=hbase:meta,,1.1588230740}
2024-12-03T01:12:48,721 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:12:48,722 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:12:48,722 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:12:48,722 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:12:48,722 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:12:48,722 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 3d508b286c6b64ce9e95f4f9bd1763f0, acc8878667541fe51516b373c2b39ae9
2024-12-03T01:12:48,722 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.81 KB heapSize=5.32 KB
2024-12-03T01:12:48,742 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/.tmp/info/566cc395466d4b91ba0292c32ecdc44e is 45, key is default/info:d/1733188261070/Put/seqid=0
2024-12-03T01:12:48,751 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/info/f1fa1fb68e7b4ea0b4a471c1c9601f9f is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0./info:regioninfo/1733188261694/Put/seqid=0
2024-12-03T01:12:48,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741853_1029 (size=5037)
2024-12-03T01:12:48,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741853_1029 (size=5037)
2024-12-03T01:12:48,757 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/.tmp/info/566cc395466d4b91ba0292c32ecdc44e
2024-12-03T01:12:48,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741854_1030 (size=8172)
2024-12-03T01:12:48,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741854_1030 (size=8172)
2024-12-03T01:12:48,766 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.59 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/info/f1fa1fb68e7b4ea0b4a471c1c9601f9f
2024-12-03T01:12:48,769 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/.tmp/info/566cc395466d4b91ba0292c32ecdc44e as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/info/566cc395466d4b91ba0292c32ecdc44e
2024-12-03T01:12:48,775 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T01:12:48,775 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T01:12:48,786 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/info/566cc395466d4b91ba0292c32ecdc44e, entries=2, sequenceid=6, filesize=4.9 K
2024-12-03T01:12:48,787 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for acc8878667541fe51516b373c2b39ae9 in 66ms, sequenceid=6, compaction requested=false
2024-12-03T01:12:48,797 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/namespace/acc8878667541fe51516b373c2b39ae9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T01:12:48,800 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:12:48,800 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for acc8878667541fe51516b373c2b39ae9:

2024-12-03T01:12:48,801 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733188260514.acc8878667541fe51516b373c2b39ae9.
2024-12-03T01:12:48,801 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 3d508b286c6b64ce9e95f4f9bd1763f0, disabling compactions & flushes
2024-12-03T01:12:48,801 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:48,801 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:48,801 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0. after waiting 0 ms
2024-12-03T01:12:48,801 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:48,801 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 3d508b286c6b64ce9e95f4f9bd1763f0 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-03T01:12:48,809 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/b2ef002ae8344be2a893e5408ab842c8 is 1080, key is row0029/info:/1733188362706/Put/seqid=0
2024-12-03T01:12:48,811 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/table/6a9a968fd3da40c9bd628fc854984f41 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733188261706/Put/seqid=0
2024-12-03T01:12:48,827 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741855_1031 (size=8193)
2024-12-03T01:12:48,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741855_1031 (size=8193)
2024-12-03T01:12:48,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741856_1032 (size=5452)
2024-12-03T01:12:48,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741856_1032 (size=5452)
2024-12-03T01:12:48,831 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/b2ef002ae8344be2a893e5408ab842c8
2024-12-03T01:12:48,835 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=232 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/table/6a9a968fd3da40c9bd628fc854984f41
2024-12-03T01:12:48,846 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/.tmp/info/b2ef002ae8344be2a893e5408ab842c8 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/b2ef002ae8344be2a893e5408ab842c8
2024-12-03T01:12:48,852 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/info/f1fa1fb68e7b4ea0b4a471c1c9601f9f as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/info/f1fa1fb68e7b4ea0b4a471c1c9601f9f
2024-12-03T01:12:48,857 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/b2ef002ae8344be2a893e5408ab842c8, entries=3, sequenceid=48, filesize=8.0 K
2024-12-03T01:12:48,862 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 3d508b286c6b64ce9e95f4f9bd1763f0 in 60ms, sequenceid=48, compaction requested=true
2024-12-03T01:12:48,876 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688] to archive
2024-12-03T01:12:48,877 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/info/f1fa1fb68e7b4ea0b4a471c1c9601f9f, entries=20, sequenceid=14, filesize=8.0 K
2024-12-03T01:12:48,880 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/.tmp/table/6a9a968fd3da40c9bd628fc854984f41 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/table/6a9a968fd3da40c9bd628fc854984f41
2024-12-03T01:12:48,885 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:12:48,891 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/457d846862fb4de58792cf8f30a877f9
2024-12-03T01:12:48,894 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/1a671f3bc7d044778977f4a663cae88e
2024-12-03T01:12:48,895 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/table/6a9a968fd3da40c9bd628fc854984f41, entries=4, sequenceid=14, filesize=5.3 K
2024-12-03T01:12:48,896 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688 to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/archive/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/info/6c95d1ac48774405b5bb6983368d2688
2024-12-03T01:12:48,897 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~2.81 KB/2882, heapSize ~5.04 KB/5160, currentSize=0 B/0 for 1588230740 in 175ms, sequenceid=14, compaction requested=false
2024-12-03T01:12:48,922 DEBUG [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 3d508b286c6b64ce9e95f4f9bd1763f0
2024-12-03T01:12:48,928 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-03T01:12:48,929 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T01:12:48,930 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:12:48,930 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:12:48,930 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T01:12:48,937 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/data/default/TestLogRolling-testSlowSyncLogRolling/3d508b286c6b64ce9e95f4f9bd1763f0/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-03T01:12:48,938 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:48,938 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 3d508b286c6b64ce9e95f4f9bd1763f0:

2024-12-03T01:12:48,939 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733188261298.3d508b286c6b64ce9e95f4f9bd1763f0.
2024-12-03T01:12:49,122 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,41559,1733188258576; all regions closed.
2024-12-03T01:12:49,124 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576
2024-12-03T01:12:49,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741834_1010 (size=4330)
2024-12-03T01:12:49,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741834_1010 (size=4330)
2024-12-03T01:12:49,532 DEBUG [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs
2024-12-03T01:12:49,532 INFO  [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C41559%2C1733188258576.meta:.meta(num 1733188260254)
2024-12-03T01:12:49,532 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/WALs/1a7b1a4db65e,41559,1733188258576
2024-12-03T01:12:49,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741851_1027 (size=13066)
2024-12-03T01:12:49,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741851_1027 (size=13066)
2024-12-03T01:12:49,542 DEBUG [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(1071): Moved 3 WAL file(s) to /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/oldWALs
2024-12-03T01:12:49,542 INFO  [RS:0;1a7b1a4db65e:41559 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C41559%2C1733188258576:(num 1733188348792)
2024-12-03T01:12:49,542 DEBUG [RS:0;1a7b1a4db65e:41559 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:12:49,542 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:12:49,542 INFO  [RS:0;1a7b1a4db65e:41559 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T01:12:49,543 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:12:49,543 INFO  [RS:0;1a7b1a4db65e:41559 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41559
2024-12-03T01:12:49,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,41559,1733188258576
2024-12-03T01:12:49,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:12:49,549 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,41559,1733188258576]
2024-12-03T01:12:49,549 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,41559,1733188258576; numProcessing=1
2024-12-03T01:12:49,551 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,41559,1733188258576 already deleted, retry=false
2024-12-03T01:12:49,551 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,41559,1733188258576 expired; onlineServers=0
2024-12-03T01:12:49,551 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,38995,1733188257813' *****
2024-12-03T01:12:49,551 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:12:49,551 DEBUG [M:0;1a7b1a4db65e:38995 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7ad3d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:12:49,551 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,38995,1733188257813
2024-12-03T01:12:49,551 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,38995,1733188257813; all regions closed.
2024-12-03T01:12:49,551 DEBUG [M:0;1a7b1a4db65e:38995 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:12:49,552 DEBUG [M:0;1a7b1a4db65e:38995 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:12:49,552 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:12:49,552 DEBUG [M:0;1a7b1a4db65e:38995 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:12:49,552 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188259541 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188259541,5,FailOnTimeoutGroup]
2024-12-03T01:12:49,552 INFO  [M:0;1a7b1a4db65e:38995 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:12:49,552 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188259542 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188259542,5,FailOnTimeoutGroup]
2024-12-03T01:12:49,552 DEBUG [M:0;1a7b1a4db65e:38995 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:12:49,552 INFO  [M:0;1a7b1a4db65e:38995 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:12:49,553 INFO  [M:0;1a7b1a4db65e:38995 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:12:49,553 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:12:49,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:12:49,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:49,556 DEBUG [M:0;1a7b1a4db65e:38995 {}] zookeeper.ZKUtil(347): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:12:49,556 WARN  [M:0;1a7b1a4db65e:38995 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:12:49,556 INFO  [M:0;1a7b1a4db65e:38995 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:12:49,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:12:49,556 INFO  [M:0;1a7b1a4db65e:38995 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:12:49,557 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:12:49,557 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:49,557 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:49,557 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:12:49,557 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:49,557 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.22 KB heapSize=50.15 KB
2024-12-03T01:12:49,581 DEBUG [M:0;1a7b1a4db65e:38995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90f146f7b5d0462a89024391adfbfac6 is 82, key is hbase:meta,,1/info:regioninfo/1733188260395/Put/seqid=0
2024-12-03T01:12:49,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741857_1033 (size=5672)
2024-12-03T01:12:49,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741857_1033 (size=5672)
2024-12-03T01:12:49,601 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90f146f7b5d0462a89024391adfbfac6
2024-12-03T01:12:49,632 DEBUG [M:0;1a7b1a4db65e:38995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/719f686c1d814c0f9909ec385cfcc1f5 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733188261714/Put/seqid=0
2024-12-03T01:12:49,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741858_1034 (size=6427)
2024-12-03T01:12:49,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741858_1034 (size=6427)
2024-12-03T01:12:49,640 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.62 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/719f686c1d814c0f9909ec385cfcc1f5
2024-12-03T01:12:49,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:12:49,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41559-0x10175dfef2e0001, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:12:49,649 INFO  [RS:0;1a7b1a4db65e:41559 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,41559,1733188258576; zookeeper connection closed.
2024-12-03T01:12:49,650 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7729b830 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7729b830
2024-12-03T01:12:49,650 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T01:12:49,651 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 719f686c1d814c0f9909ec385cfcc1f5
2024-12-03T01:12:49,671 DEBUG [M:0;1a7b1a4db65e:38995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5da0d2ed30c4139af9be057a66c7516 is 69, key is 1a7b1a4db65e,41559,1733188258576/rs:state/1733188259681/Put/seqid=0
2024-12-03T01:12:49,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741859_1035 (size=5156)
2024-12-03T01:12:49,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741859_1035 (size=5156)
2024-12-03T01:12:49,682 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5da0d2ed30c4139af9be057a66c7516
2024-12-03T01:12:49,706 DEBUG [M:0;1a7b1a4db65e:38995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ace385ceac8146ee8ecacd384efc86fc is 52, key is load_balancer_on/state:d/1733188261223/Put/seqid=0
2024-12-03T01:12:49,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741860_1036 (size=5056)
2024-12-03T01:12:49,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741860_1036 (size=5056)
2024-12-03T01:12:49,761 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:12:50,116 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ace385ceac8146ee8ecacd384efc86fc
2024-12-03T01:12:50,131 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90f146f7b5d0462a89024391adfbfac6 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/90f146f7b5d0462a89024391adfbfac6
2024-12-03T01:12:50,140 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/90f146f7b5d0462a89024391adfbfac6, entries=8, sequenceid=104, filesize=5.5 K
2024-12-03T01:12:50,141 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/719f686c1d814c0f9909ec385cfcc1f5 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/719f686c1d814c0f9909ec385cfcc1f5
2024-12-03T01:12:50,148 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 719f686c1d814c0f9909ec385cfcc1f5
2024-12-03T01:12:50,149 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/719f686c1d814c0f9909ec385cfcc1f5, entries=11, sequenceid=104, filesize=6.3 K
2024-12-03T01:12:50,150 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f5da0d2ed30c4139af9be057a66c7516 as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f5da0d2ed30c4139af9be057a66c7516
2024-12-03T01:12:50,163 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f5da0d2ed30c4139af9be057a66c7516, entries=1, sequenceid=104, filesize=5.0 K
2024-12-03T01:12:50,167 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ace385ceac8146ee8ecacd384efc86fc as hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ace385ceac8146ee8ecacd384efc86fc
2024-12-03T01:12:50,176 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ace385ceac8146ee8ecacd384efc86fc, entries=1, sequenceid=104, filesize=4.9 K
2024-12-03T01:12:50,177 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.22 KB/41185, heapSize ~50.09 KB/51288, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 620ms, sequenceid=104, compaction requested=false
2024-12-03T01:12:50,186 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:50,186 DEBUG [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:12:50,188 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/MasterData/WALs/1a7b1a4db65e,38995,1733188257813
2024-12-03T01:12:50,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43133 is added to blk_1073741830_1006 (size=48486)
2024-12-03T01:12:50,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42289 is added to blk_1073741830_1006 (size=48486)
2024-12-03T01:12:50,593 INFO  [M:0;1a7b1a4db65e:38995 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:12:50,593 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:12:50,594 INFO  [M:0;1a7b1a4db65e:38995 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38995
2024-12-03T01:12:50,595 DEBUG [M:0;1a7b1a4db65e:38995 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,38995,1733188257813 already deleted, retry=false
2024-12-03T01:12:50,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:12:50,697 INFO  [M:0;1a7b1a4db65e:38995 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,38995,1733188257813; zookeeper connection closed.
2024-12-03T01:12:50,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38995-0x10175dfef2e0000, quorum=127.0.0.1:49500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:12:50,702 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@163cfad6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:12:50,705 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f952caa{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:12:50,705 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:12:50,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6eb1b261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:12:50,705 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4debea22{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,STOPPED}
2024-12-03T01:12:50,709 WARN  [BP-218709567-172.17.0.2-1733188254599 heartbeating to localhost/127.0.0.1:45713 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:12:50,709 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:12:50,709 WARN  [BP-218709567-172.17.0.2-1733188254599 heartbeating to localhost/127.0.0.1:45713 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-218709567-172.17.0.2-1733188254599 (Datanode Uuid b736c9d1-7911-41c1-9f05-0b8863b05fee) service to localhost/127.0.0.1:45713
2024-12-03T01:12:50,709 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:12:50,711 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data3/current/BP-218709567-172.17.0.2-1733188254599 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:12:50,711 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data4/current/BP-218709567-172.17.0.2-1733188254599 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:12:50,712 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:12:50,718 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6aad8790{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:12:50,719 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@587d1dca{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:12:50,719 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:12:50,719 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4ce9e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:12:50,719 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2276bd44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,STOPPED}
2024-12-03T01:12:50,722 WARN  [BP-218709567-172.17.0.2-1733188254599 heartbeating to localhost/127.0.0.1:45713 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:12:50,722 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:12:50,722 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:12:50,722 WARN  [BP-218709567-172.17.0.2-1733188254599 heartbeating to localhost/127.0.0.1:45713 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-218709567-172.17.0.2-1733188254599 (Datanode Uuid 33a7291a-d48a-407d-bf7b-0c0429d7dbb6) service to localhost/127.0.0.1:45713
2024-12-03T01:12:50,723 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data1/current/BP-218709567-172.17.0.2-1733188254599 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:12:50,723 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/cluster_7f4b66fd-c301-ad36-49a5-7bf802de8ed3/dfs/data/data2/current/BP-218709567-172.17.0.2-1733188254599 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:12:50,723 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:12:50,734 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5682c4d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:12:50,735 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ff1a6c1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:12:50,735 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:12:50,735 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74468826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:12:50,735 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@88aab13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir/,STOPPED}
2024-12-03T01:12:50,747 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:12:50,789 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:12:50,799 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=61 (was 12)
Potentially hanging thread: region-location-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:45713 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: master/1a7b1a4db65e:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/1a7b1a4db65e:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45713
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/1a7b1a4db65e:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45713
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45713
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45713
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@47fd615d
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:45713 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:45713
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:45713 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=411 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=571 (was 853), ProcessCount=11 (was 11), AvailableMemoryMB=2934 (was 3894)
2024-12-03T01:12:50,807 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=62, OpenFileDescriptor=411, MaxFileDescriptor=1048576, SystemLoadAverage=571, ProcessCount=11, AvailableMemoryMB=2934
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.log.dir so I do NOT create it in target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/70daf0e7-7577-903a-c299-b0fd9bd020a6/hadoop.tmp.dir so I do NOT create it in target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de, deleteOnExit=true
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/test.cache.data in system properties and HBase conf
2024-12-03T01:12:50,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:12:50,809 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:12:50,809 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:12:50,810 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:12:50,830 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:12:50,938 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:12:50,944 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:12:50,952 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:12:50,952 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:12:50,952 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:12:50,953 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:12:50,954 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39523ff9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:12:50,955 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1101772a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:12:51,102 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@69d3d453{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-33003-hadoop-hdfs-3_4_1-tests_jar-_-any-17962061007639829653/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:12:51,102 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@52ca9eab{HTTP/1.1, (http/1.1)}{localhost:33003}
2024-12-03T01:12:51,103 INFO  [Time-limited test {}] server.Server(415): Started @119102ms
2024-12-03T01:12:51,123 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:12:51,210 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:12:51,216 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:12:51,221 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:12:51,221 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:12:51,221 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:12:51,222 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e9304e6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:12:51,222 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59774852{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:12:51,356 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b51cf53{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-38851-hadoop-hdfs-3_4_1-tests_jar-_-any-5243940806992100068/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:12:51,357 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a2efd9{HTTP/1.1, (http/1.1)}{localhost:38851}
2024-12-03T01:12:51,357 INFO  [Time-limited test {}] server.Server(415): Started @119357ms
2024-12-03T01:12:51,360 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:12:51,416 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:12:51,427 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:12:51,434 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:12:51,434 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:12:51,434 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:12:51,435 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b628890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:12:51,435 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29f7fced{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:12:51,515 WARN  [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data1/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:12:51,517 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data2/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:12:51,595 WARN  [Thread-430 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:12:51,599 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c586cce{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-33283-hadoop-hdfs-3_4_1-tests_jar-_-any-15250670156298178951/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:12:51,601 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d899dad{HTTP/1.1, (http/1.1)}{localhost:33283}
2024-12-03T01:12:51,601 INFO  [Time-limited test {}] server.Server(415): Started @119601ms
2024-12-03T01:12:51,604 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:12:51,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0fb9c7101b33d7e with lease ID 0xb7a415a98df2d0d8: Processing first storage report for DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798 from datanode DatanodeRegistration(127.0.0.1:38263, datanodeUuid=c29cd568-1ae2-4782-be2c-e65927242f48, infoPort=43715, infoSecurePort=0, ipcPort=37849, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:12:51,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0fb9c7101b33d7e with lease ID 0xb7a415a98df2d0d8: from storage DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798 node DatanodeRegistration(127.0.0.1:38263, datanodeUuid=c29cd568-1ae2-4782-be2c-e65927242f48, infoPort=43715, infoSecurePort=0, ipcPort=37849, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:12:51,610 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0fb9c7101b33d7e with lease ID 0xb7a415a98df2d0d8: Processing first storage report for DS-592da816-7499-4a04-ae31-3268d02d5fb1 from datanode DatanodeRegistration(127.0.0.1:38263, datanodeUuid=c29cd568-1ae2-4782-be2c-e65927242f48, infoPort=43715, infoSecurePort=0, ipcPort=37849, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:12:51,610 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0fb9c7101b33d7e with lease ID 0xb7a415a98df2d0d8: from storage DS-592da816-7499-4a04-ae31-3268d02d5fb1 node DatanodeRegistration(127.0.0.1:38263, datanodeUuid=c29cd568-1ae2-4782-be2c-e65927242f48, infoPort=43715, infoSecurePort=0, ipcPort=37849, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:12:51,730 WARN  [Thread-477 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data3/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:12:51,735 WARN  [Thread-478 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data4/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:12:51,766 WARN  [Thread-466 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:12:51,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8bbcecfbf77fbd9 with lease ID 0xb7a415a98df2d0d9: Processing first storage report for DS-67caa125-8283-4927-b785-4e24e62d4ebb from datanode DatanodeRegistration(127.0.0.1:41891, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=34715, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:12:51,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8bbcecfbf77fbd9 with lease ID 0xb7a415a98df2d0d9: from storage DS-67caa125-8283-4927-b785-4e24e62d4ebb node DatanodeRegistration(127.0.0.1:41891, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=34715, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:12:51,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8bbcecfbf77fbd9 with lease ID 0xb7a415a98df2d0d9: Processing first storage report for DS-b7b84440-7a4f-475f-a8bb-c7314ea6d37a from datanode DatanodeRegistration(127.0.0.1:41891, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=34715, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:12:51,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8bbcecfbf77fbd9 with lease ID 0xb7a415a98df2d0d9: from storage DS-b7b84440-7a4f-475f-a8bb-c7314ea6d37a node DatanodeRegistration(127.0.0.1:41891, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=34715, infoSecurePort=0, ipcPort=35471, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:12:51,848 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c
2024-12-03T01:12:51,858 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/zookeeper_0, clientPort=58035, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:12:51,861 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58035
2024-12-03T01:12:51,861 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:51,864 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:51,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:12:51,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:12:51,899 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354 with version=8
2024-12-03T01:12:51,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:51,903 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:12:51,904 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:12:51,904 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:12:51,905 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:35881
2024-12-03T01:12:51,905 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:51,907 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:51,911 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:35881 connecting to ZooKeeper ensemble=127.0.0.1:58035
2024-12-03T01:12:51,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:358810x0, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:12:51,932 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35881-0x10175e1b01f0000 connected
2024-12-03T01:12:52,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:12:52,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:12:52,016 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:12:52,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35881
2024-12-03T01:12:52,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35881
2024-12-03T01:12:52,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35881
2024-12-03T01:12:52,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35881
2024-12-03T01:12:52,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35881
2024-12-03T01:12:52,022 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354, hbase.cluster.distributed=false
2024-12-03T01:12:52,052 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:12:52,052 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:52,052 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:52,052 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:12:52,052 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:52,053 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:12:52,053 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:12:52,053 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:12:52,054 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:37929
2024-12-03T01:12:52,054 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:12:52,055 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:12:52,056 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:52,060 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:52,066 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:37929 connecting to ZooKeeper ensemble=127.0.0.1:58035
2024-12-03T01:12:52,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:379290x0, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:12:52,080 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37929-0x10175e1b01f0001 connected
2024-12-03T01:12:52,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:12:52,094 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:12:52,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:12:52,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37929
2024-12-03T01:12:52,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37929
2024-12-03T01:12:52,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37929
2024-12-03T01:12:52,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37929
2024-12-03T01:12:52,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37929
2024-12-03T01:12:52,136 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:12:52,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:12:52,146 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:12:52,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:12:52,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,148 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:12:52,149 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,35881,1733188371902 from backup master directory
2024-12-03T01:12:52,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:12:52,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:12:52,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:12:52,151 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:12:52,151 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,155 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:35881
2024-12-03T01:12:52,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:12:52,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:12:52,170 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/hbase.id with ID: d9d61786-7e64-4ec8-911d-b37774aab83c
2024-12-03T01:12:52,184 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:52,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:12:52,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:12:52,198 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:12:52,199 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:12:52,200 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:12:52,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:12:52,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:12:52,618 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store
2024-12-03T01:12:52,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:12:52,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:12:52,627 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:52,627 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:12:52,627 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:52,628 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:52,628 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:12:52,628 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:52,628 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:12:52,628 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:12:52,629 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/.initializing
2024-12-03T01:12:52,629 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,633 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C35881%2C1733188371902, suffix=, logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902, archiveDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/oldWALs, maxLogs=10
2024-12-03T01:12:52,634 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C35881%2C1733188371902.1733188372633
2024-12-03T01:12:52,640 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633
2024-12-03T01:12:52,640 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34715:34715),(127.0.0.1/127.0.0.1:43715:43715)]
2024-12-03T01:12:52,640 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:12:52,641 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:52,641 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:12:52,641 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:12:52,642 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:12:52,644 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:12:52,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,645 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:52,645 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:12:52,647 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:12:52,647 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,647 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:12:52,648 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:12:52,649 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:12:52,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,650 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:12:52,650 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:12:52,653 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:12:52,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,654 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:12:52,655 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:12:52,656 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:12:52,658 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:12:52,659 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:12:52,662 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:12:52,662 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860688, jitterRate=0.09442150592803955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:12:52,665 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:12:52,666 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:12:52,670 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59acd2c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:12:52,672 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:12:52,672 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:12:52,672 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:12:52,672 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:12:52,673 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec
2024-12-03T01:12:52,674 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-03T01:12:52,674 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:12:52,677 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:12:52,685 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:12:52,693 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:12:52,694 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:12:52,695 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:12:52,696 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:12:52,696 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:12:52,697 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:12:52,701 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:12:52,702 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:12:52,704 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:12:52,706 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:12:52,707 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:12:52,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:12:52,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:12:52,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,710 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,35881,1733188371902, sessionid=0x10175e1b01f0000, setting cluster-up flag (Was=false)
2024-12-03T01:12:52,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,724 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:12:52,725 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:52,739 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:12:52,740 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:52,744 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:12:52,744 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:12:52,744 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,35881,1733188371902
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:12:52,745 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188402753
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:12:52,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:12:52,754 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:12:52,754 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,755 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:12:52,755 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:12:52,755 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:12:52,755 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:12:52,755 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:12:52,755 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188372755,5,FailOnTimeoutGroup]
2024-12-03T01:12:52,756 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188372755,5,FailOnTimeoutGroup]
2024-12-03T01:12:52,756 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,756 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:12:52,756 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,756 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,756 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,756 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:12:52,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:12:52,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:12:52,824 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:12:52,825 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354
2024-12-03T01:12:52,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:12:52,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:12:52,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:52,849 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:12:52,851 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:12:52,852 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:52,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:12:52,854 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:37929
2024-12-03T01:12:52,855 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1008): ClusterId : d9d61786-7e64-4ec8-911d-b37774aab83c
2024-12-03T01:12:52,855 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:12:52,855 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:12:52,855 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:52,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:12:52,859 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:12:52,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:52,859 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:12:52,859 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:12:52,859 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:52,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/1588230740
2024-12-03T01:12:52,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/1588230740
2024-12-03T01:12:52,862 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:12:52,863 DEBUG [RS:0;1a7b1a4db65e:37929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254a49ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:12:52,863 DEBUG [RS:0;1a7b1a4db65e:37929 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536fa812, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:12:52,864 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:12:52,864 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:12:52,864 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:12:52,865 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,35881,1733188371902 with isa=1a7b1a4db65e/172.17.0.2:37929, startcode=1733188372050
2024-12-03T01:12:52,865 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:12:52,865 DEBUG [RS:0;1a7b1a4db65e:37929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:12:52,867 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:12:52,875 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56011, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:12:52,875 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35881 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,875 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35881 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,878 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354
2024-12-03T01:12:52,878 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43249
2024-12-03T01:12:52,878 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:12:52,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:12:52,881 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:12:52,882 DEBUG [RS:0;1a7b1a4db65e:37929 {}] zookeeper.ZKUtil(111): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,882 WARN  [RS:0;1a7b1a4db65e:37929 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:12:52,882 INFO  [RS:0;1a7b1a4db65e:37929 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:12:52,882 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,883 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751922, jitterRate=-0.043882086873054504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:12:52,884 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,37929,1733188372050]
2024-12-03T01:12:52,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:12:52,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:12:52,884 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:12:52,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:12:52,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:12:52,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:12:52,893 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:12:52,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:12:52,894 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:12:52,894 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:12:52,895 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:12:52,896 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:12:52,900 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:12:52,904 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:12:52,904 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:12:52,916 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:12:52,921 INFO  [RS:0;1a7b1a4db65e:37929 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:12:52,921 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,921 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:12:52,923 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,923 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,923 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,923 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,923 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:12:52,924 DEBUG [RS:0;1a7b1a4db65e:37929 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:12:52,929 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,929 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,929 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,929 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,936 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,37929,1733188372050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:12:52,964 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:12:52,964 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,37929,1733188372050-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:52,990 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.Replication(204): 1a7b1a4db65e,37929,1733188372050 started
2024-12-03T01:12:52,990 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,37929,1733188372050, RpcServer on 1a7b1a4db65e/172.17.0.2:37929, sessionid=0x10175e1b01f0001
2024-12-03T01:12:52,990 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:12:52,990 DEBUG [RS:0;1a7b1a4db65e:37929 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,990 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,37929,1733188372050'
2024-12-03T01:12:52,990 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:12:52,993 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:12:52,996 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:12:52,996 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:12:52,996 DEBUG [RS:0;1a7b1a4db65e:37929 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:52,997 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,37929,1733188372050'
2024-12-03T01:12:52,997 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:12:52,997 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:12:53,001 DEBUG [RS:0;1a7b1a4db65e:37929 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:12:53,001 INFO  [RS:0;1a7b1a4db65e:37929 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:12:53,001 INFO  [RS:0;1a7b1a4db65e:37929 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:12:53,051 WARN  [1a7b1a4db65e:35881 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-03T01:12:53,103 INFO  [RS:0;1a7b1a4db65e:37929 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C37929%2C1733188372050, suffix=, logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050, archiveDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs, maxLogs=32
2024-12-03T01:12:53,106 INFO  [RS:0;1a7b1a4db65e:37929 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188373105
2024-12-03T01:12:53,134 INFO  [RS:0;1a7b1a4db65e:37929 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105
2024-12-03T01:12:53,134 DEBUG [RS:0;1a7b1a4db65e:37929 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34715:34715),(127.0.0.1/127.0.0.1:43715:43715)]
2024-12-03T01:12:53,301 DEBUG [1a7b1a4db65e:35881 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:12:53,302 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,303 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,37929,1733188372050, state=OPENING
2024-12-03T01:12:53,305 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:12:53,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:53,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:53,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:12:53,307 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,37929,1733188372050}]
2024-12-03T01:12:53,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:12:53,462 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,462 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:12:53,465 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:12:53,469 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:12:53,470 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:12:53,472 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C37929%2C1733188372050.meta, suffix=.meta, logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050, archiveDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs, maxLogs=32
2024-12-03T01:12:53,474 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
2024-12-03T01:12:53,479 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
2024-12-03T01:12:53,479 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43715:43715),(127.0.0.1/127.0.0.1:34715:34715)]
2024-12-03T01:12:53,480 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:12:53,480 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:12:53,480 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:12:53,480 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:12:53,480 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:12:53,480 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:53,481 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:12:53,481 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:12:53,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:12:53,484 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:12:53,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:53,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:53,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:12:53,485 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:12:53,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:53,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:53,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:12:53,487 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:12:53,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:53,488 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:12:53,489 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/1588230740
2024-12-03T01:12:53,490 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/meta/1588230740
2024-12-03T01:12:53,492 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:12:53,493 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:12:53,494 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877209, jitterRate=0.11542962491512299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:12:53,495 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:12:53,496 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188373462
2024-12-03T01:12:53,499 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:12:53,499 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:12:53,500 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,500 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,37929,1733188372050, state=OPEN
2024-12-03T01:12:53,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:12:53,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:12:53,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:12:53,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:12:53,511 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:12:53,512 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,37929,1733188372050 in 201 msec
2024-12-03T01:12:53,514 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:12:53,514 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 617 msec
2024-12-03T01:12:53,516 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 773 msec
2024-12-03T01:12:53,517 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188373517, completionTime=-1
2024-12-03T01:12:53,517 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:12:53,517 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:12:53,518 DEBUG [hconnection-0x634771de-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:12:53,519 INFO  [RS-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44356, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:12:53,520 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:12:53,520 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188433520
2024-12-03T01:12:53,521 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188493520
2024-12-03T01:12:53,521 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:35881, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:12:53,527 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:12:53,528 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:12:53,529 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:12:53,530 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:12:53,530 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:53,531 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:12:53,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:12:53,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:12:53,543 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 85448ed9a38888aa02c9cb429da4d961, NAME => 'hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354
2024-12-03T01:12:53,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:12:53,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 85448ed9a38888aa02c9cb429da4d961, disabling compactions & flushes
2024-12-03T01:12:53,553 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961. after waiting 0 ms
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,553 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,553 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 85448ed9a38888aa02c9cb429da4d961:

2024-12-03T01:12:53,555 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:12:53,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188373555"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188373555"}]},"ts":"1733188373555"}
2024-12-03T01:12:53,558 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:12:53,559 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:12:53,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188373559"}]},"ts":"1733188373559"}
2024-12-03T01:12:53,561 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:12:53,567 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=85448ed9a38888aa02c9cb429da4d961, ASSIGN}]
2024-12-03T01:12:53,568 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=85448ed9a38888aa02c9cb429da4d961, ASSIGN
2024-12-03T01:12:53,569 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=85448ed9a38888aa02c9cb429da4d961, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,37929,1733188372050; forceNewPlan=false, retain=false
2024-12-03T01:12:53,720 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=85448ed9a38888aa02c9cb429da4d961, regionState=OPENING, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,723 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 85448ed9a38888aa02c9cb429da4d961, server=1a7b1a4db65e,37929,1733188372050}]
2024-12-03T01:12:53,802 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:53,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:53,877 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,883 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 85448ed9a38888aa02c9cb429da4d961, NAME => 'hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:12:53,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:53,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,884 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,888 INFO  [StoreOpener-85448ed9a38888aa02c9cb429da4d961-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 85448ed9a38888aa02c9cb429da4d961 
2024-12-03T01:12:53,890 INFO  [StoreOpener-85448ed9a38888aa02c9cb429da4d961-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85448ed9a38888aa02c9cb429da4d961 columnFamilyName info
2024-12-03T01:12:53,891 DEBUG [StoreOpener-85448ed9a38888aa02c9cb429da4d961-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:53,891 INFO  [StoreOpener-85448ed9a38888aa02c9cb429da4d961-1 {}] regionserver.HStore(327): Store=85448ed9a38888aa02c9cb429da4d961/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:12:53,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,893 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,896 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:12:53,899 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:12:53,900 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 85448ed9a38888aa02c9cb429da4d961; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=783412, jitterRate=-0.0038412660360336304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:12:53,901 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 85448ed9a38888aa02c9cb429da4d961:

2024-12-03T01:12:53,903 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961., pid=6, masterSystemTime=1733188373877
2024-12-03T01:12:53,905 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,905 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:12:53,906 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=85448ed9a38888aa02c9cb429da4d961, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:53,912 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:12:53,912 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 85448ed9a38888aa02c9cb429da4d961, server=1a7b1a4db65e,37929,1733188372050 in 186 msec
2024-12-03T01:12:53,915 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:12:53,915 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=85448ed9a38888aa02c9cb429da4d961, ASSIGN in 345 msec
2024-12-03T01:12:53,916 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:12:53,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188373916"}]},"ts":"1733188373916"}
2024-12-03T01:12:53,918 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:12:53,922 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:12:53,924 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 395 msec
2024-12-03T01:12:53,930 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:12:53,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:12:53,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:53,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:12:53,938 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:12:53,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:12:53,952 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 13 msec
2024-12-03T01:12:53,960 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:12:53,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:12:53,974 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec
2024-12-03T01:12:53,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:12:53,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:12:53,989 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.838sec
2024-12-03T01:12:53,989 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:12:53,989 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:12:53,989 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:12:53,989 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:12:53,990 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:12:53,990 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:12:53,990 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:12:53,993 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:12:53,993 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:12:53,993 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35881,1733188371902-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,036 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x15f71bfd to 127.0.0.1:58035 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7faa2a92
2024-12-03T01:12:54,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb20afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:12:54,041 DEBUG [hconnection-0x67463d7c-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:12:54,044 INFO  [RS-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44372, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:12:54,046 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,35881,1733188371902
2024-12-03T01:12:54,047 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:54,050 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:12:54,068 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:12:54,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:54,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:54,068 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:12:54,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:12:54,069 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:12:54,069 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:12:54,069 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:12:54,070 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:35129
2024-12-03T01:12:54,070 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:12:54,073 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:12:54,074 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:54,076 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:12:54,079 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:35129 connecting to ZooKeeper ensemble=127.0.0.1:58035
2024-12-03T01:12:54,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351290x0, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:12:54,083 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35129-0x10175e1b01f0003 connected
2024-12-03T01:12:54,083 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:12:54,084 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-03T01:12:54,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:12:54,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35129
2024-12-03T01:12:54,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35129
2024-12-03T01:12:54,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35129
2024-12-03T01:12:54,086 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35129
2024-12-03T01:12:54,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35129
2024-12-03T01:12:54,088 DEBUG [pool-282-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-03T01:12:54,108 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1a7b1a4db65e:35129
2024-12-03T01:12:54,112 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1008): ClusterId : d9d61786-7e64-4ec8-911d-b37774aab83c
2024-12-03T01:12:54,112 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:12:54,117 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:12:54,117 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:12:54,120 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:12:54,121 DEBUG [RS:1;1a7b1a4db65e:35129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391f2be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:12:54,122 DEBUG [RS:1;1a7b1a4db65e:35129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b7fd74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:12:54,122 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:12:54,122 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:12:54,122 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:12:54,123 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,35881,1733188371902 with isa=1a7b1a4db65e/172.17.0.2:35129, startcode=1733188374067
2024-12-03T01:12:54,123 DEBUG [RS:1;1a7b1a4db65e:35129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:12:54,126 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60673, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:12:54,126 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35881 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,126 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35881 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,129 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354
2024-12-03T01:12:54,129 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43249
2024-12-03T01:12:54,129 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:12:54,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:12:54,133 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,35129,1733188374067]
2024-12-03T01:12:54,135 DEBUG [RS:1;1a7b1a4db65e:35129 {}] zookeeper.ZKUtil(111): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,136 WARN  [RS:1;1a7b1a4db65e:35129 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:12:54,136 INFO  [RS:1;1a7b1a4db65e:35129 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:12:54,136 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,140 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:12:54,140 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:12:54,143 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:12:54,143 INFO  [RS:1;1a7b1a4db65e:35129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:12:54,143 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,144 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:12:54,145 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,145 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,146 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,146 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,146 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:12:54,146 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:12:54,146 DEBUG [RS:1;1a7b1a4db65e:35129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:12:54,146 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,146 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,146 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,146 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,146 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35129,1733188374067-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:12:54,169 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:12:54,170 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,35129,1733188374067-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:12:54,194 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.Replication(204): 1a7b1a4db65e,35129,1733188374067 started
2024-12-03T01:12:54,194 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,35129,1733188374067, RpcServer on 1a7b1a4db65e/172.17.0.2:35129, sessionid=0x10175e1b01f0003
2024-12-03T01:12:54,195 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:12:54,195 DEBUG [RS:1;1a7b1a4db65e:35129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(3355): Started new server=Thread[RS:1;1a7b1a4db65e:35129,5,FailOnTimeoutGroup]
2024-12-03T01:12:54,195 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,35129,1733188374067'
2024-12-03T01:12:54,195 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:12:54,195 INFO  [Time-limited test {}] wal.TestLogRolling(191): Replication=2
2024-12-03T01:12:54,195 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:12:54,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T01:12:54,196 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:12:54,196 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:12:54,196 DEBUG [RS:1;1a7b1a4db65e:35129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,35129,1733188374067
2024-12-03T01:12:54,196 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,35129,1733188374067'
2024-12-03T01:12:54,196 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:12:54,197 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:12:54,197 DEBUG [RS:1;1a7b1a4db65e:35129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:12:54,197 INFO  [RS:1;1a7b1a4db65e:35129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:12:54,197 INFO  [RS:1;1a7b1a4db65e:35129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:12:54,198 INFO  [RS-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39046, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T01:12:54,200 WARN  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T01:12:54,200 WARN  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T01:12:54,201 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:12:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T01:12:54,203 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:12:54,204 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:54,204 INFO  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 9
2024-12-03T01:12:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:12:54,205 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:12:54,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741837_1013 (size=393)
2024-12-03T01:12:54,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741837_1013 (size=393)
2024-12-03T01:12:54,217 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4b99be8732eddce4ec2a4c71b945ea2c, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354
2024-12-03T01:12:54,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38263 is added to blk_1073741838_1014 (size=76)
2024-12-03T01:12:54,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41891 is added to blk_1073741838_1014 (size=76)
2024-12-03T01:12:54,300 INFO  [RS:1;1a7b1a4db65e:35129 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C35129%2C1733188374067, suffix=, logDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067, archiveDir=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs, maxLogs=32
2024-12-03T01:12:54,302 INFO  [RS:1;1a7b1a4db65e:35129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C35129%2C1733188374067.1733188374302
2024-12-03T01:12:54,314 INFO  [RS:1;1a7b1a4db65e:35129 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067/1a7b1a4db65e%2C35129%2C1733188374067.1733188374302
2024-12-03T01:12:54,314 DEBUG [RS:1;1a7b1a4db65e:35129 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43715:43715),(127.0.0.1/127.0.0.1:34715:34715)]
2024-12-03T01:12:54,329 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:12:54,332 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:54,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:54,627 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:54,628 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1681): Closing 4b99be8732eddce4ec2a4c71b945ea2c, disabling compactions & flushes
2024-12-03T01:12:54,628 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,628 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,628 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c. after waiting 0 ms
2024-12-03T01:12:54,628 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,628 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,628 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:12:54,630 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:12:54,630 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733188374630"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188374630"}]},"ts":"1733188374630"}
2024-12-03T01:12:54,633 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:12:54,634 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:12:54,634 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188374634"}]},"ts":"1733188374634"}
2024-12-03T01:12:54,637 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-03T01:12:54,647 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(202): Hosts are {1a7b1a4db65e=0} racks are {/default-rack=0}
2024-12-03T01:12:54,651 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 is on host 0
2024-12-03T01:12:54,651 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 is on host 0
2024-12-03T01:12:54,651 INFO  [PEWorker-2 {}] balancer.BalancerClusterState(314): server 0 is on rack 0
2024-12-03T01:12:54,651 INFO  [PEWorker-2 {}] balancer.BalancerClusterState(314): server 1 is on rack 0
2024-12-03T01:12:54,651 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1
2024-12-03T01:12:54,652 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=4b99be8732eddce4ec2a4c71b945ea2c, ASSIGN}]
2024-12-03T01:12:54,655 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=4b99be8732eddce4ec2a4c71b945ea2c, ASSIGN
2024-12-03T01:12:54,656 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=4b99be8732eddce4ec2a4c71b945ea2c, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,37929,1733188372050; forceNewPlan=false, retain=false
2024-12-03T01:12:54,807 INFO  [1a7b1a4db65e:35881 {}] balancer.BaseLoadBalancer(546): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-03T01:12:54,808 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=4b99be8732eddce4ec2a4c71b945ea2c, regionState=OPENING, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:54,811 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 4b99be8732eddce4ec2a4c71b945ea2c, server=1a7b1a4db65e,37929,1733188372050}]
2024-12-03T01:12:54,964 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:54,969 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,969 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 4b99be8732eddce4ec2a4c71b945ea2c, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:12:54,970 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,970 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:12:54,970 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,970 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,972 INFO  [StoreOpener-4b99be8732eddce4ec2a4c71b945ea2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4b99be8732eddce4ec2a4c71b945ea2c 
2024-12-03T01:12:54,974 INFO  [StoreOpener-4b99be8732eddce4ec2a4c71b945ea2c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b99be8732eddce4ec2a4c71b945ea2c columnFamilyName info
2024-12-03T01:12:54,974 DEBUG [StoreOpener-4b99be8732eddce4ec2a4c71b945ea2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:12:54,974 INFO  [StoreOpener-4b99be8732eddce4ec2a4c71b945ea2c-1 {}] regionserver.HStore(327): Store=4b99be8732eddce4ec2a4c71b945ea2c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:12:54,975 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,976 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,978 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:12:54,981 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:12:54,982 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 4b99be8732eddce4ec2a4c71b945ea2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=748270, jitterRate=-0.048526614904403687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:12:54,983 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:12:54,984 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c., pid=11, masterSystemTime=1733188374964
2024-12-03T01:12:54,986 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,986 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:12:54,987 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=4b99be8732eddce4ec2a4c71b945ea2c, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,37929,1733188372050
2024-12-03T01:12:54,992 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-03T01:12:54,992 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 4b99be8732eddce4ec2a4c71b945ea2c, server=1a7b1a4db65e,37929,1733188372050 in 179 msec
2024-12-03T01:12:54,995 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T01:12:54,995 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=4b99be8732eddce4ec2a4c71b945ea2c, ASSIGN in 340 msec
2024-12-03T01:12:54,996 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:12:54,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188374996"}]},"ts":"1733188374996"}
2024-12-03T01:12:54,998 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-03T01:12:55,001 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:12:55,003 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 800 msec
2024-12-03T01:12:58,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T01:12:58,331 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-03T01:12:58,333 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-03T01:12:59,386 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:12:59,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:59,413 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:12:59,424 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T01:12:59,425 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-03T01:12:59,427 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-03T01:13:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35881 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:13:04,207 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath, procId: 9 completed
2024-12-03T01:13:04,211 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-03T01:13:04,211 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:04,225 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:04,229 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:04,229 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:04,229 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:04,229 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:13:04,230 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a8a53d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:04,230 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a5b3132{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:04,350 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49dbde5b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-35427-hadoop-hdfs-3_4_1-tests_jar-_-any-11163215179895471658/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:04,350 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10289dc3{HTTP/1.1, (http/1.1)}{localhost:35427}
2024-12-03T01:13:04,350 INFO  [Time-limited test {}] server.Server(415): Started @132350ms
2024-12-03T01:13:04,352 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:04,389 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:04,393 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:04,394 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:04,394 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:04,394 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:13:04,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126966d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:04,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@779f3433{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:04,466 WARN  [Thread-634 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,466 WARN  [Thread-635 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,491 WARN  [Thread-614 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:04,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x269f393e8f54de5b with lease ID 0xb7a415a98df2d0da: Processing first storage report for DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097 from datanode DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x269f393e8f54de5b with lease ID 0xb7a415a98df2d0da: from storage DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097 node DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x269f393e8f54de5b with lease ID 0xb7a415a98df2d0da: Processing first storage report for DS-1ce7d0cf-6c76-4429-a029-61cf077cd4e1 from datanode DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x269f393e8f54de5b with lease ID 0xb7a415a98df2d0da: from storage DS-1ce7d0cf-6c76-4429-a029-61cf077cd4e1 node DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,519 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18d7a785{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-34205-hadoop-hdfs-3_4_1-tests_jar-_-any-15800905430250141368/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:04,519 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a13bbe3{HTTP/1.1, (http/1.1)}{localhost:34205}
2024-12-03T01:13:04,519 INFO  [Time-limited test {}] server.Server(415): Started @132519ms
2024-12-03T01:13:04,521 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:04,564 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:04,568 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:04,569 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:04,569 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:04,569 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:13:04,572 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56ac9d38{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:04,572 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11c134c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:04,637 WARN  [Thread-669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data7/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,637 WARN  [Thread-670 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data8/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,663 WARN  [Thread-649 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:04,665 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x931fc5aaede006d6 with lease ID 0xb7a415a98df2d0db: Processing first storage report for DS-b61d928e-bc46-4507-a913-88b4b493b77e from datanode DatanodeRegistration(127.0.0.1:34901, datanodeUuid=d16b2492-622b-4be3-b7b8-16b933650f32, infoPort=45797, infoSecurePort=0, ipcPort=37395, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,665 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x931fc5aaede006d6 with lease ID 0xb7a415a98df2d0db: from storage DS-b61d928e-bc46-4507-a913-88b4b493b77e node DatanodeRegistration(127.0.0.1:34901, datanodeUuid=d16b2492-622b-4be3-b7b8-16b933650f32, infoPort=45797, infoSecurePort=0, ipcPort=37395, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,665 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x931fc5aaede006d6 with lease ID 0xb7a415a98df2d0db: Processing first storage report for DS-6d5b7b36-d0ad-4598-85d9-fc9881cabf9b from datanode DatanodeRegistration(127.0.0.1:34901, datanodeUuid=d16b2492-622b-4be3-b7b8-16b933650f32, infoPort=45797, infoSecurePort=0, ipcPort=37395, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,665 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x931fc5aaede006d6 with lease ID 0xb7a415a98df2d0db: from storage DS-6d5b7b36-d0ad-4598-85d9-fc9881cabf9b node DatanodeRegistration(127.0.0.1:34901, datanodeUuid=d16b2492-622b-4be3-b7b8-16b933650f32, infoPort=45797, infoSecurePort=0, ipcPort=37395, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,695 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5037bd43{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-38965-hadoop-hdfs-3_4_1-tests_jar-_-any-1420147326900442053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:04,696 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2cb953bd{HTTP/1.1, (http/1.1)}{localhost:38965}
2024-12-03T01:13:04,696 INFO  [Time-limited test {}] server.Server(415): Started @132696ms
2024-12-03T01:13:04,698 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:04,819 WARN  [Thread-695 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data9/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,820 WARN  [Thread-696 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data10/current/BP-845475697-172.17.0.2-1733188370852/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:04,843 WARN  [Thread-684 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:04,846 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa1994a16255a2e02 with lease ID 0xb7a415a98df2d0dc: Processing first storage report for DS-801650d8-bedc-4b6c-a542-9aa254c4890d from datanode DatanodeRegistration(127.0.0.1:45033, datanodeUuid=e0ed39e1-49de-4044-9866-f88af5795a5a, infoPort=34647, infoSecurePort=0, ipcPort=38771, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,847 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa1994a16255a2e02 with lease ID 0xb7a415a98df2d0dc: from storage DS-801650d8-bedc-4b6c-a542-9aa254c4890d node DatanodeRegistration(127.0.0.1:45033, datanodeUuid=e0ed39e1-49de-4044-9866-f88af5795a5a, infoPort=34647, infoSecurePort=0, ipcPort=38771, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,847 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa1994a16255a2e02 with lease ID 0xb7a415a98df2d0dc: Processing first storage report for DS-5aaa9d79-72bc-4d79-83e5-9e8030fe668a from datanode DatanodeRegistration(127.0.0.1:45033, datanodeUuid=e0ed39e1-49de-4044-9866-f88af5795a5a, infoPort=34647, infoSecurePort=0, ipcPort=38771, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852)
2024-12-03T01:13:04,847 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa1994a16255a2e02 with lease ID 0xb7a415a98df2d0dc: from storage DS-5aaa9d79-72bc-4d79-83e5-9e8030fe668a node DatanodeRegistration(127.0.0.1:45033, datanodeUuid=e0ed39e1-49de-4044-9866-f88af5795a5a, infoPort=34647, infoSecurePort=0, ipcPort=38771, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:04,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c586cce{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:04,923 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d899dad{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:04,923 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:04,923 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29f7fced{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:04,924 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b628890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:04,920 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,920 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015
java.io.IOException: Bad response ERROR for BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015 from datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,920 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,926 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067/1a7b1a4db65e%2C35129%2C1733188374067.1733188374302 block BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK], DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:04,920 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,926 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 block BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:04,927 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:04,927 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta block BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK], DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:04,928 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:04,928 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 block BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:04,928 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid 18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36) service to localhost/127.0.0.1:43249
2024-12-03T01:13:04,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:50510 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41891:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50510 dst: /127.0.0.1:41891
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 50071 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,927 WARN  [PacketResponder: BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41891] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,929 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:04,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_874188856_22 at /127.0.0.1:50524 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:41891:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50524 dst: /127.0.0.1:41891
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 49387 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:50506 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41891:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50506 dst: /127.0.0.1:41891
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 50057 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,928 WARN  [PacketResponder: BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41891] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-544292685_22 at /127.0.0.1:50490 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41891:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50490 dst: /127.0.0.1:41891
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 50077 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,930 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data3/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:04,930 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-544292685_22 at /127.0.0.1:45362 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45362 dst: /127.0.0.1:38263
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,929 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:45396 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:38263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45396 dst: /127.0.0.1:38263
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,932 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:04,931 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data4/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:04,930 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:45404 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45404 dst: /127.0.0.1:38263
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,931 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_874188856_22 at /127.0.0.1:45454 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015] {}] datanode.DataXceiver(331): 127.0.0.1:38263:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45454 dst: /127.0.0.1:38263
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:04,934 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 block BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,934 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta block BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,934 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 block BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,936 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b51cf53{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:04,936 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a2efd9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:04,936 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:04,937 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59774852{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:04,937 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e9304e6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:04,938 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067/1a7b1a4db65e%2C35129%2C1733188374067.1733188374302 block BP-845475697-172.17.0.2-1733188370852:blk_1073741839_1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1015
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,939 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:04,939 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:04,939 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid c29cd568-1ae2-4782-be2c-e65927242f48) service to localhost/127.0.0.1:43249
2024-12-03T01:13:04,939 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:04,940 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data1/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:04,940 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data2/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:04,940 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:04,948 WARN  [RS:0;1a7b1a4db65e:37929.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=4, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,948 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C37929%2C1733188372050:(num 1733188373105) roll requested
2024-12-03T01:13:04,948 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188384948
2024-12-03T01:13:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37929 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37929 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:44372 deadline: 1733188394947, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-03T01:13:04,961 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=4, requesting roll of WAL
2024-12-03T01:13:04,961 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 with entries=4, filesize=959 B; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948
2024-12-03T01:13:04,965 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45797:45797),(127.0.0.1/127.0.0.1:34647:34647)]
2024-12-03T01:13:04,965 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:04,965 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,965 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:04,966 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-03T01:13:04,966 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-03T01:13:04,967 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105
2024-12-03T01:13:04,969 WARN  [IPC Server handler 2 on default port 43249 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741833_1009
2024-12-03T01:13:04,971 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 after 4ms
2024-12-03T01:13:08,972 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 after 4005ms
2024-12-03T01:13:17,065 INFO  [Time-limited test {}] wal.TestLogRolling(243): log.getCurrentFileName(): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948
2024-12-03T01:13:17,066 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:17,067 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948 block BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:17,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:58372 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:34901:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58372 dst: /127.0.0.1:34901
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:17,067 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:50496 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:45033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50496 dst: /127.0.0.1:45033
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:17,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18d7a785{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:17,069 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a13bbe3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:17,069 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:17,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@779f3433{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:17,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126966d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:17,073 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:17,073 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:17,073 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:17,073 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid d16b2492-622b-4be3-b7b8-16b933650f32) service to localhost/127.0.0.1:43249
2024-12-03T01:13:17,074 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data7/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:17,074 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data8/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:17,075 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:17,077 WARN  [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]
2024-12-03T01:13:17,077 WARN  [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]
2024-12-03T01:13:17,078 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C37929%2C1733188372050:(num 1733188384948) roll requested
2024-12-03T01:13:17,078 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188397078
2024-12-03T01:13:17,082 WARN  [Thread-714 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38263
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:17,082 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:47772 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023 to mirror 127.0.0.1:38263
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:17,082 WARN  [Thread-714 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]) is bad.
2024-12-03T01:13:17,082 WARN  [Thread-714 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023
2024-12-03T01:13:17,082 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:47772 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:17,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:47772 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47772 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:17,086 WARN  [Thread-714 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]
2024-12-03T01:13:17,089 WARN  [Thread-714 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:17,090 WARN  [Thread-714 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:17,090 WARN  [Thread-714 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741842_1024
2024-12-03T01:13:17,090 WARN  [Thread-714 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:17,092 WARN  [Thread-714 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:17,092 WARN  [Thread-714 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:17,092 WARN  [Thread-714 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741843_1025
2024-12-03T01:13:17,093 WARN  [Thread-714 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
2024-12-03T01:13:17,108 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078
2024-12-03T01:13:17,108 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36171:36171),(127.0.0.1/127.0.0.1:34647:34647)]
2024-12-03T01:13:17,108 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:17,108 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948 is not closed yet, will try archiving it next time
2024-12-03T01:13:17,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45033 is added to blk_1073741840_1022 (size=2431)
2024-12-03T01:13:17,511 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:18,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741840_1022 (size=2431)
2024-12-03T01:13:21,082 WARN  [ResponseProcessor for block BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026
java.io.IOException: Bad response ERROR for BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026 from datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,082 WARN  [DataStreamer for file /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078 block BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:21,082 WARN  [PacketResponder: BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45033] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:47776 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47776 dst: /127.0.0.1:39367
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,083 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:41572 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:45033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41572 dst: /127.0.0.1:45033
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5037bd43{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:21,085 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2cb953bd{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:21,085 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:21,085 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11c134c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:21,085 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56ac9d38{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:21,088 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:21,088 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:21,088 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:21,088 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid e0ed39e1-49de-4044-9866-f88af5795a5a) service to localhost/127.0.0.1:43249
2024-12-03T01:13:21,089 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data9/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:21,089 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data10/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:21,089 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:21,092 WARN  [sync.4 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]
2024-12-03T01:13:21,092 WARN  [sync.4 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]
2024-12-03T01:13:21,092 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C37929%2C1733188372050:(num 1733188397078) roll requested
2024-12-03T01:13:21,092 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188401092
2024-12-03T01:13:21,095 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,095 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:21,095 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741845_1028
2024-12-03T01:13:21,096 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
2024-12-03T01:13:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37929 {}] regionserver.HRegion(8581): Flush requested on 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:13:21,097 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 4b99be8732eddce4ec2a4c71b945ea2c 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:13:21,098 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38263
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,098 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43422 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029 to mirror 127.0.0.1:38263
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,099 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]) is bad.
2024-12-03T01:13:21,099 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029
2024-12-03T01:13:21,099 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43422 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:21,099 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43422 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43422 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,099 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]
2024-12-03T01:13:21,104 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43430 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030 to mirror 127.0.0.1:45033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,104 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:21,104 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030
2024-12-03T01:13:21,104 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43430 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:21,105 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:21,107 WARN  [Thread-727 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,107 WARN  [Thread-727 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:21,107 WARN  [Thread-727 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741848_1031
2024-12-03T01:13:21,108 WARN  [Thread-727 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:21,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43430 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43430 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,109 WARN  [IPC Server handler 4 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T01:13:21,109 WARN  [IPC Server handler 4 on default port 43249 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T01:13:21,109 WARN  [IPC Server handler 4 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T01:13:21,113 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078 with entries=12, filesize=12.96 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401092
2024-12-03T01:13:21,113 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36171:36171)]
2024-12-03T01:13:21,113 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,114 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,114 WARN  [sync.1 {}] wal.FSHLog(750): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]
2024-12-03T01:13:21,114 WARN  [sync.1 {}] wal.FSHLog(721): Requesting log roll because of low replication, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]
2024-12-03T01:13:21,114 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C37929%2C1733188372050:(num 1733188401092) roll requested
2024-12-03T01:13:21,114 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188401114
2024-12-03T01:13:21,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741844_1027 (size=13275)
2024-12-03T01:13:21,116 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,118 WARN  [Thread-734 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,118 WARN  [Thread-734 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:21,118 WARN  [Thread-734 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741850_1033
2024-12-03T01:13:21,119 WARN  [Thread-734 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:21,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/bcf5d3abe4624f88b4eeaba9f3ccdbb3 is 1080, key is row0002/info:/1733188397076/Put/seqid=0
2024-12-03T01:13:21,120 WARN  [Thread-734 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,120 WARN  [Thread-734 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:21,121 WARN  [Thread-734 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741851_1034
2024-12-03T01:13:21,121 WARN  [Thread-734 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:21,122 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,122 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK], DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]) is bad.
2024-12-03T01:13:21,122 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741852_1035
2024-12-03T01:13:21,123 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]
2024-12-03T01:13:21,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43450 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036 to mirror 127.0.0.1:38263
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,125 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43450 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:21,125 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43450 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43450 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,125 WARN  [Thread-734 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38263
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,125 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43462 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037 to mirror 127.0.0.1:34901
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,125 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34901
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,126 WARN  [Thread-734 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]) is bad.
2024-12-03T01:13:21,126 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43462 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T01:13:21,126 WARN  [Thread-734 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741853_1036
2024-12-03T01:13:21,126 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:21,126 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037
2024-12-03T01:13:21,126 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43462 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43462 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,127 WARN  [Thread-734 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]
2024-12-03T01:13:21,128 WARN  [Thread-734 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,128 WARN  [Thread-734 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:21,128 WARN  [Thread-734 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741855_1038
2024-12-03T01:13:21,129 WARN  [Thread-734 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
2024-12-03T01:13:21,129 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:21,129 WARN  [IPC Server handler 0 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T01:13:21,129 WARN  [IPC Server handler 0 on default port 43249 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T01:13:21,130 WARN  [IPC Server handler 0 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T01:13:21,132 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41891
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,132 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43474 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040 to mirror 127.0.0.1:41891
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,133 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]) is bad.
2024-12-03T01:13:21,133 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040
2024-12-03T01:13:21,133 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43474 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T01:13:21,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43474 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43474 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,133 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41891,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]
2024-12-03T01:13:21,136 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401092 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401114
2024-12-03T01:13:21,136 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36171:36171)]
2024-12-03T01:13:21,136 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,136 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401092 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,136 WARN  [Thread-729 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:21,136 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43482 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041 to mirror 127.0.0.1:45033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,137 WARN  [Thread-729 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:21,137 WARN  [Thread-729 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041
2024-12-03T01:13:21,137 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43482 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-03T01:13:21,137 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:43482 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43482 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,137 WARN  [Thread-729 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:21,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741849_1032 (size=1261)
2024-12-03T01:13:21,140 WARN  [IPC Server handler 2 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-03T01:13:21,140 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 is not closed yet, will try archiving it next time
2024-12-03T01:13:21,140 WARN  [IPC Server handler 2 on default port 43249 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-03T01:13:21,140 WARN  [IPC Server handler 2 on default port 43249 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-03T01:13:21,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741859_1042 (size=10347)
2024-12-03T01:13:21,316 WARN  [sync.3 {}] wal.FSHLog(760): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-03T01:13:21,497 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1ca5db8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741844_1027 to 127.0.0.1:41891 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,497 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@21696f92[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741849_1032 to 127.0.0.1:41891 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:21,533 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:21,541 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:21,546 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:21,546 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:21,546 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:13:21,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@356979bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:21,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39d43134{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:21,553 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/bcf5d3abe4624f88b4eeaba9f3ccdbb3
2024-12-03T01:13:21,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/bcf5d3abe4624f88b4eeaba9f3ccdbb3 as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/info/bcf5d3abe4624f88b4eeaba9f3ccdbb3
2024-12-03T01:13:21,583 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/info/bcf5d3abe4624f88b4eeaba9f3ccdbb3, entries=5, sequenceid=12, filesize=10.1 K
2024-12-03T01:13:21,584 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=8.40 KB/8606 for 4b99be8732eddce4ec2a4c71b945ea2c in 487ms, sequenceid=12, compaction requested=false
2024-12-03T01:13:21,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:13:21,689 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59dcb886{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/java.io.tmpdir/jetty-localhost-42797-hadoop-hdfs-3_4_1-tests_jar-_-any-6862167642196186148/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:21,690 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3dcf703a{HTTP/1.1, (http/1.1)}{localhost:42797}
2024-12-03T01:13:21,690 INFO  [Time-limited test {}] server.Server(415): Started @149689ms
2024-12-03T01:13:21,692 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:21,836 WARN  [Thread-763 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:21,847 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:13:21,849 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x767135568118fb3d with lease ID 0xb7a415a98df2d0dd: from storage DS-67caa125-8283-4927-b785-4e24e62d4ebb node DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 8, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:13:21,849 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x767135568118fb3d with lease ID 0xb7a415a98df2d0dd: from storage DS-b7b84440-7a4f-475f-a8bb-c7314ea6d37a node DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:22,497 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@21696f92[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741859_1042 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:22,754 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=96, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,754 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C35881%2C1733188371902:(num 1733188372633) roll requested
2024-12-03T01:13:22,755 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C35881%2C1733188371902.1733188402754
2024-12-03T01:13:22,755 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,755 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,760 WARN  [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-544292685_22 at /127.0.0.1:43498 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6]'}, localName='127.0.0.1:39367', datanodeUuid='ed9489eb-e2bf-41cf-8b29-036e4f6c150b', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043 to mirror 127.0.0.1:45033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:22,760 WARN  [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:22,760 WARN  [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043
2024-12-03T01:13:22,760 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-544292685_22 at /127.0.0.1:43498 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:22,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-544292685_22 at /127.0.0.1:43498 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:39367:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43498 dst: /127.0.0.1:39367
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:22,761 WARN  [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:22,762 WARN  [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,763 WARN  [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]) is bad.
2024-12-03T01:13:22,763 WARN  [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741861_1044
2024-12-03T01:13:22,763 WARN  [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]
2024-12-03T01:13:22,765 WARN  [Thread-783 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,765 WARN  [Thread-783 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:22,765 WARN  [Thread-783 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741862_1045
2024-12-03T01:13:22,765 WARN  [Thread-783 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:22,776 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=96, requesting roll of WAL
2024-12-03T01:13:22,776 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 with entries=93, filesize=46.04 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188402754
2024-12-03T01:13:22,777 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36171:36171),(127.0.0.1/127.0.0.1:33615:33615)]
2024-12-03T01:13:22,777 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 is not closed yet, will try archiving it next time
2024-12-03T01:13:22,777 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,777 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:22,777 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633
2024-12-03T01:13:22,778 WARN  [IPC Server handler 0 on default port 43249 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 has not been closed. Lease recovery is in progress. RecoveryId = 1047 for block blk_1073741830_1006
2024-12-03T01:13:22,778 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 after 1ms
2024-12-03T01:13:25,863 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:13:25,866 INFO  [RS-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34394, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:13:26,121 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:13:26,123 INFO  [RS-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:13:26,780 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902/1a7b1a4db65e%2C35881%2C1733188371902.1733188372633 after 4003ms
2024-12-03T01:13:31,867 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@314a6fa2 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:38263,null,null])
java.net.ConnectException: Call From 1a7b1a4db65e/172.17.0.2 to localhost:37849 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-03T01:13:31,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741833_1021 (size=959)
2024-12-03T01:13:31,996 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948 to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs/1a7b1a4db65e%2C37929%2C1733188372050.1733188384948
2024-12-03T01:13:33,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741837_1013 (size=393)
2024-12-03T01:13:33,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:13:33,994 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T01:13:33,995 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T01:13:34,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@33b0ce46[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741831_1007 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:35,856 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b1fb98[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741829_1005 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:36,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@33b0ce46[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741828_1004 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:36,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b1fb98[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741832_1008 to 127.0.0.1:45033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:37,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b1fb98[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741826_1002 to 127.0.0.1:45033 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:37,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@33b0ce46[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741833_1021 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:39,845 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@33b0ce46[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741827_1003 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:39,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:13:39,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 10347
2024-12-03T01:13:40,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@33b0ce46[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741836_1012 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:40,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741838_1014 (size=76)
2024-12-03T01:13:40,861 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.1733188420861
2024-12-03T01:13:40,864 WARN  [Thread-799 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,864 WARN  [Thread-799 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741864_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:40,864 WARN  [Thread-799 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741864_1048
2024-12-03T01:13:40,865 WARN  [Thread-799 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:40,872 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401114 with entries=2, filesize=1.57 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188420861
2024-12-03T01:13:40,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33615:33615),(127.0.0.1/127.0.0.1:36171:36171)]
2024-12-03T01:13:40,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401114 is not closed yet, will try archiving it next time
2024-12-03T01:13:40,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741856_1039 (size=1618)
2024-12-03T01:13:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37929 {}] regionserver.HRegion(8581): Flush requested on 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:13:40,875 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 4b99be8732eddce4ec2a4c71b945ea2c 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-03T01:13:40,876 INFO  [sync.2 {}] wal.FSHLog(777): LowReplication-Roller was enabled.
2024-12-03T01:13:40,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/ea1640b7b3f54c98a9c50688e7bd49b0 is 1080, key is row0007/info:/1733188401098/Put/seqid=0
2024-12-03T01:13:40,881 WARN  [Thread-805 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,882 WARN  [Thread-805 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741866_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK], DatanodeInfoWithStorage[127.0.0.1:36469,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:40,882 WARN  [Thread-805 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741866_1050
2024-12-03T01:13:40,882 WARN  [Thread-805 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:40,883 WARN  [Thread-805 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,884 WARN  [Thread-805 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741867_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK], DatanodeInfoWithStorage[127.0.0.1:39367,DS-f5d1a944-6c9a-4943-8bd2-0ba61f847097,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:40,884 WARN  [Thread-805 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741867_1051
2024-12-03T01:13:40,884 WARN  [Thread-805 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:40,888 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:13:40,888 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-03T01:13:40,888 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x15f71bfd to 127.0.0.1:58035
2024-12-03T01:13:40,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:40,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:13:40,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=282242223, stopped=false
2024-12-03T01:13:40,888 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,35881,1733188371902
2024-12-03T01:13:40,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741868_1052 (size=13583)
2024-12-03T01:13:40,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741868_1052 (size=13583)
2024-12-03T01:13:40,890 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/ea1640b7b3f54c98a9c50688e7bd49b0
2024-12-03T01:13:40,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:13:40,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:13:40,891 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:13:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:13:40,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:40,892 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:13:40,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:40,893 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:13:40,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:13:40,894 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,37929,1733188372050' *****
2024-12-03T01:13:40,894 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:13:40,894 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,35129,1733188374067' *****
2024-12-03T01:13:40,894 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:13:40,894 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:13:40,895 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:13:40,895 INFO  [RS:1;1a7b1a4db65e:35129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:13:40,895 INFO  [RS:1;1a7b1a4db65e:35129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:13:40,895 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:13:40,895 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,35129,1733188374067
2024-12-03T01:13:40,895 DEBUG [RS:1;1a7b1a4db65e:35129 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:40,895 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,35129,1733188374067; all regions closed.
2024-12-03T01:13:40,895 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,35129,1733188374067
2024-12-03T01:13:40,895 WARN  [WAL-Shutdown-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,896 ERROR [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1664): Shutdown / close of WAL failed: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
2024-12-03T01:13:40,896 DEBUG [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1665): Shutdown / close exception details:
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,896 DEBUG [RS:1;1a7b1a4db65e:35129 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:40,896 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:13:40,896 INFO  [RS:1;1a7b1a4db65e:35129 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T01:13:40,897 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:13:40,897 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:13:40,897 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:13:40,897 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:13:40,897 INFO  [RS:1;1a7b1a4db65e:35129 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:35129
2024-12-03T01:13:40,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,35129,1733188374067
2024-12-03T01:13:40,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:13:40,900 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,35129,1733188374067]
2024-12-03T01:13:40,900 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,35129,1733188374067; numProcessing=1
2024-12-03T01:13:40,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/.tmp/info/ea1640b7b3f54c98a9c50688e7bd49b0 as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/info/ea1640b7b3f54c98a9c50688e7bd49b0
2024-12-03T01:13:40,904 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,35129,1733188374067 already deleted, retry=false
2024-12-03T01:13:40,904 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,35129,1733188374067 expired; onlineServers=1
2024-12-03T01:13:40,911 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/info/ea1640b7b3f54c98a9c50688e7bd49b0, entries=8, sequenceid=24, filesize=13.3 K
2024-12-03T01:13:40,912 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~9.46 KB/9682, heapSize ~10.36 KB/10608, currentSize=9.46 KB/9684 for 4b99be8732eddce4ec2a4c71b945ea2c in 38ms, sequenceid=24, compaction requested=false
2024-12-03T01:13:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:13:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=23.4 K, sizeToCheck=16.0 K
2024-12-03T01:13:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:13:40,913 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/default/TestLogRolling-testLogRollOnDatanodeDeath/4b99be8732eddce4ec2a4c71b945ea2c/info/ea1640b7b3f54c98a9c50688e7bd49b0 because midkey is the same as first or last row
2024-12-03T01:13:40,913 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:13:40,913 INFO  [RS:0;1a7b1a4db65e:37929 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:13:40,913 INFO  [RS:0;1a7b1a4db65e:37929 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:13:40,913 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3579): Received CLOSE for 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3579): Received CLOSE for 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,37929,1733188372050
2024-12-03T01:13:40,914 DEBUG [RS:0;1a7b1a4db65e:37929 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 85448ed9a38888aa02c9cb429da4d961, disabling compactions & flushes
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:13:40,914 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961. after waiting 0 ms
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:13:40,914 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 85448ed9a38888aa02c9cb429da4d961 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:13:40,914 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-03T01:13:40,914 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1603): Online Regions={85448ed9a38888aa02c9cb429da4d961=hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961., 1588230740=hbase:meta,,1.1588230740, 4b99be8732eddce4ec2a4c71b945ea2c=TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.}
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:13:40,914 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 4b99be8732eddce4ec2a4c71b945ea2c, 85448ed9a38888aa02c9cb429da4d961
2024-12-03T01:13:40,914 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:13:40,914 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:13:40,915 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:13:40,915 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:13:40,915 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.87 KB heapSize=5.40 KB
2024-12-03T01:13:40,915 WARN  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,915 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C37929%2C1733188372050.meta:.meta(num 1733188373474) roll requested
2024-12-03T01:13:40,915 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:13:40,916 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188420915.meta
2024-12-03T01:13:40,916 ERROR [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server 1a7b1a4db65e,37929,1733188372050: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,916 ERROR [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint]
2024-12-03T01:13:40,919 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory
2024-12-03T01:13:40,920 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC
2024-12-03T01:13:40,920 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication
2024-12-03T01:13:40,920 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server
2024-12-03T01:13:40,920 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: {
  "beans": [
    {
      "name": "java.lang:type=Memory",
      "modelerType": "sun.management.MemoryImpl",
      "ObjectPendingFinalizationCount": 0,
      "HeapMemoryUsage": {
        "committed": 1048576000,
        "init": 1048576000,
        "max": 2306867200,
        "used": 266954176
      },
      "NonHeapMemoryUsage": {
        "committed": 162398208,
        "init": 7667712,
        "max": -1,
        "used": 160695872
      },
      "Verbose": false,
      "ObjectName": "java.lang:type=Memory"
    }
  ],
  "beans": [],
  "beans": [],
  "beans": []
}
2024-12-03T01:13:40,923 WARN  [Thread-813 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45033
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,923 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60846 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data4]'}, localName='127.0.0.1:36469', datanodeUuid='18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053 to mirror 127.0.0.1:45033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:40,923 WARN  [Thread-813 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36469,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:40,924 WARN  [Thread-813 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053
2024-12-03T01:13:40,924 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60846 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:40,924 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60846 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741869_1053] {}] datanode.DataXceiver(331): 127.0.0.1:36469:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60846 dst: /127.0.0.1:36469
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:40,924 WARN  [Thread-813 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:40,925 WARN  [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35881 {}] master.MasterRpcServices(626): 1a7b1a4db65e,37929,1733188372050 reported a fatal error:
***** ABORTING region server 1a7b1a4db65e,37929,1733188372050: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
Cause:
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967)
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168)
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627)
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408)
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707)

2024-12-03T01:13:40,933 WARN  [Thread-813 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1054
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34901
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,934 WARN  [Thread-813 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36469,DS-67caa125-8283-4927-b785-4e24e62d4ebb,DISK], DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]) is bad.
2024-12-03T01:13:40,934 WARN  [Thread-813 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054
2024-12-03T01:13:40,934 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60856 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data4]'}, localName='127.0.0.1:36469', datanodeUuid='18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36', xmitsInProgress=0}:Exception transferring block BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054 to mirror 127.0.0.1:34901
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:40,934 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60856 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-03T01:13:40,934 WARN  [Thread-813 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]
2024-12-03T01:13:40,935 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1236750134_22 at /127.0.0.1:60856 [Receiving block BP-845475697-172.17.0.2-1733188370852:blk_1073741870_1054] {}] datanode.DataXceiver(331): 127.0.0.1:36469:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60856 dst: /127.0.0.1:36469
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:40,938 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/.tmp/info/e6e1c87b69cb480384cefe499f15854e is 45, key is default/info:d/1733188373943/Put/seqid=0
2024-12-03T01:13:40,939 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:13:40,940 WARN  [Thread-814 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,940 WARN  [Thread-814 {}] hdfs.DataStreamer(1731): Error Recovery for BP-845475697-172.17.0.2-1733188370852:blk_1073741872_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK], DatanodeInfoWithStorage[127.0.0.1:34901,DS-b61d928e-bc46-4507-a913-88b4b493b77e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]) is bad.
2024-12-03T01:13:40,940 WARN  [Thread-814 {}] hdfs.DataStreamer(1850): Abandoning BP-845475697-172.17.0.2-1733188370852:blk_1073741872_1056
2024-12-03T01:13:40,941 WARN  [Thread-814 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45033,DS-801650d8-bedc-4b6c-a542-9aa254c4890d,DISK]
2024-12-03T01:13:40,946 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-03T01:13:40,946 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta with entries=11, filesize=3.63 KB; new WAL /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188420915.meta
2024-12-03T01:13:40,946 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36171:36171),(127.0.0.1/127.0.0.1:33615:33615)]
2024-12-03T01:13:40,946 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta is not closed yet, will try archiving it next time
2024-12-03T01:13:40,946 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,946 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38263,DS-1d4c77d5-cd45-431d-bc9f-cbd0039ec798,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:13:40,946 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
2024-12-03T01:13:40,947 WARN  [IPC Server handler 3 on default port 43249 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta has not been closed. Lease recovery is in progress. RecoveryId = 1058 for block blk_1073741834_1010
2024-12-03T01:13:40,947 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta after 1ms
2024-12-03T01:13:40,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741873_1057 (size=5037)
2024-12-03T01:13:40,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741873_1057 (size=5037)
2024-12-03T01:13:40,950 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/.tmp/info/e6e1c87b69cb480384cefe499f15854e
2024-12-03T01:13:40,958 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/.tmp/info/e6e1c87b69cb480384cefe499f15854e as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/info/e6e1c87b69cb480384cefe499f15854e
2024-12-03T01:13:40,965 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/info/e6e1c87b69cb480384cefe499f15854e, entries=2, sequenceid=6, filesize=4.9 K
2024-12-03T01:13:40,966 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 85448ed9a38888aa02c9cb429da4d961 in 52ms, sequenceid=6, compaction requested=false
2024-12-03T01:13:40,967 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T01:13:40,967 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T01:13:40,971 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/data/hbase/namespace/85448ed9a38888aa02c9cb429da4d961/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T01:13:40,972 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 85448ed9a38888aa02c9cb429da4d961:

2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733188373527.85448ed9a38888aa02c9cb429da4d961.
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4b99be8732eddce4ec2a4c71b945ea2c, disabling compactions & flushes
2024-12-03T01:13:40,972 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c. after waiting 0 ms
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:13:40,972 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:41,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:41,001 INFO  [RS:1;1a7b1a4db65e:35129 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,35129,1733188374067; zookeeper connection closed.
2024-12-03T01:13:41,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35129-0x10175e1b01f0003, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:41,001 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@716fe42a {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@716fe42a
2024-12-03T01:13:41,115 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:13:41,115 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(3579): Received CLOSE for 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:13:41,115 DEBUG [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 4b99be8732eddce4ec2a4c71b945ea2c
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4b99be8732eddce4ec2a4c71b945ea2c, disabling compactions & flushes
2024-12-03T01:13:41,115 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:13:41,115 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c. after waiting 0 ms
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4b99be8732eddce4ec2a4c71b945ea2c:

2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740
2024-12-03T01:13:41,115 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing TestLogRolling-testLogRollOnDatanodeDeath,,1733188374200.4b99be8732eddce4ec2a4c71b945ea2c.
2024-12-03T01:13:41,275 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105 to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs/1a7b1a4db65e%2C37929%2C1733188372050.1733188373105
2024-12-03T01:13:41,277 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078 to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs/1a7b1a4db65e%2C37929%2C1733188372050.1733188397078
2024-12-03T01:13:41,278 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401092 to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs/1a7b1a4db65e%2C37929%2C1733188372050.1733188401092
2024-12-03T01:13:41,279 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.1733188401114 to hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/oldWALs/1a7b1a4db65e%2C37929%2C1733188372050.1733188401114
2024-12-03T01:13:41,315 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing
2024-12-03T01:13:41,315 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,37929,1733188372050; all regions closed.
2024-12-03T01:13:41,316 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050
2024-12-03T01:13:41,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741871_1055 (size=93)
2024-12-03T01:13:41,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741871_1055 (size=93)
2024-12-03T01:13:41,871 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4ba104d8 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:38263,null,null])
java.net.ConnectException: Call From 1a7b1a4db65e/172.17.0.2 to localhost:37849 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-03T01:13:42,149 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:13:42,498 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@21696f92[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39367, datanodeUuid=ed9489eb-e2bf-41cf-8b29-036e4f6c150b, infoPort=36171, infoSecurePort=0, ipcPort=42065, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741856_1039 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:42,846 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@b1fb98[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36469, datanodeUuid=18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36, infoPort=33615, infoSecurePort=0, ipcPort=35209, storageInfo=lv=-57;cid=testClusterID;nsid=837914474;c=1733188370852):Failed to transfer BP-845475697-172.17.0.2-1733188370852:blk_1073741830_1047 to 127.0.0.1:34901 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:44,948 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta after 4002ms
2024-12-03T01:13:45,973 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:45,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:45,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:45,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:45,999 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,000 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,321 ERROR [WAL-Shutdown-0 {}] wal.FSHLog(508): We have waited 5 seconds but the close of writer(s) doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-03T01:13:46,321 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050
2024-12-03T01:13:46,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741865_1049 (size=13280)
2024-12-03T01:13:46,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741865_1049 (size=13280)
2024-12-03T01:13:46,324 DEBUG [RS:0;1a7b1a4db65e:37929 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:46,325 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:13:46,325 INFO  [RS:0;1a7b1a4db65e:37929 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-03T01:13:46,325 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:13:46,325 INFO  [RS:0;1a7b1a4db65e:37929 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:37929
2024-12-03T01:13:46,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,37929,1733188372050
2024-12-03T01:13:46,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:13:46,330 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,37929,1733188372050]
2024-12-03T01:13:46,330 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,37929,1733188372050; numProcessing=2
2024-12-03T01:13:46,332 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,37929,1733188372050 already deleted, retry=false
2024-12-03T01:13:46,332 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,37929,1733188372050 expired; onlineServers=0
2024-12-03T01:13:46,332 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,35881,1733188371902' *****
2024-12-03T01:13:46,332 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:13:46,332 DEBUG [M:0;1a7b1a4db65e:35881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b91a1c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:13:46,332 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,35881,1733188371902
2024-12-03T01:13:46,332 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,35881,1733188371902; all regions closed.
2024-12-03T01:13:46,332 DEBUG [M:0;1a7b1a4db65e:35881 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:13:46,332 DEBUG [M:0;1a7b1a4db65e:35881 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:13:46,332 DEBUG [M:0;1a7b1a4db65e:35881 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:13:46,333 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188372755 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188372755,5,FailOnTimeoutGroup]
2024-12-03T01:13:46,333 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188372755 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188372755,5,FailOnTimeoutGroup]
2024-12-03T01:13:46,333 INFO  [M:0;1a7b1a4db65e:35881 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:13:46,333 DEBUG [M:0;1a7b1a4db65e:35881 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:13:46,333 INFO  [M:0;1a7b1a4db65e:35881 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:13:46,333 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:13:46,333 INFO  [M:0;1a7b1a4db65e:35881 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:13:46,333 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:13:46,334 DEBUG [M:0;1a7b1a4db65e:35881 {}] zookeeper.ZKUtil(347): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:13:46,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:13:46,334 WARN  [M:0;1a7b1a4db65e:35881 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:13:46,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:46,334 INFO  [M:0;1a7b1a4db65e:35881 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:13:46,334 INFO  [M:0;1a7b1a4db65e:35881 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:13:46,334 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:13:46,335 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:46,335 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:46,335 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:13:46,335 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:46,335 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.08 KB heapSize=49.29 KB
2024-12-03T01:13:46,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:13:46,354 DEBUG [M:0;1a7b1a4db65e:35881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6531455a3e49466e844a21afc25fdd65 is 82, key is hbase:meta,,1/info:regioninfo/1733188373499/Put/seqid=0
2024-12-03T01:13:46,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741874_1059 (size=5672)
2024-12-03T01:13:46,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741874_1059 (size=5672)
2024-12-03T01:13:46,366 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6531455a3e49466e844a21afc25fdd65
2024-12-03T01:13:46,389 DEBUG [M:0;1a7b1a4db65e:35881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c3c54378fa44931a45135e7a9bd711a is 774, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733188375002/Put/seqid=0
2024-12-03T01:13:46,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741875_1060 (size=7465)
2024-12-03T01:13:46,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741875_1060 (size=7465)
2024-12-03T01:13:46,395 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.41 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c3c54378fa44931a45135e7a9bd711a
2024-12-03T01:13:46,415 DEBUG [M:0;1a7b1a4db65e:35881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f599d91578d478b8a7a07391d2af18e is 69, key is 1a7b1a4db65e,35129,1733188374067/rs:state/1733188374127/Put/seqid=0
2024-12-03T01:13:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741876_1061 (size=5224)
2024-12-03T01:13:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741876_1061 (size=5224)
2024-12-03T01:13:46,421 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f599d91578d478b8a7a07391d2af18e
2024-12-03T01:13:46,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:46,430 INFO  [RS:0;1a7b1a4db65e:37929 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,37929,1733188372050; zookeeper connection closed.
2024-12-03T01:13:46,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37929-0x10175e1b01f0001, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:46,431 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e8045ed {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e8045ed
2024-12-03T01:13:46,431 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-03T01:13:46,441 DEBUG [M:0;1a7b1a4db65e:35881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0cd2315335d94d80b999f857db532613 is 52, key is load_balancer_on/state:d/1733188374049/Put/seqid=0
2024-12-03T01:13:46,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741877_1062 (size=5056)
2024-12-03T01:13:46,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741877_1062 (size=5056)
2024-12-03T01:13:46,447 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0cd2315335d94d80b999f857db532613
2024-12-03T01:13:46,453 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6531455a3e49466e844a21afc25fdd65 as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6531455a3e49466e844a21afc25fdd65
2024-12-03T01:13:46,459 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6531455a3e49466e844a21afc25fdd65, entries=8, sequenceid=97, filesize=5.5 K
2024-12-03T01:13:46,460 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5c3c54378fa44931a45135e7a9bd711a as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c3c54378fa44931a45135e7a9bd711a
2024-12-03T01:13:46,465 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5c3c54378fa44931a45135e7a9bd711a, entries=11, sequenceid=97, filesize=7.3 K
2024-12-03T01:13:46,466 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f599d91578d478b8a7a07391d2af18e as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f599d91578d478b8a7a07391d2af18e
2024-12-03T01:13:46,471 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f599d91578d478b8a7a07391d2af18e, entries=2, sequenceid=97, filesize=5.1 K
2024-12-03T01:13:46,472 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0cd2315335d94d80b999f857db532613 as hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0cd2315335d94d80b999f857db532613
2024-12-03T01:13:46,476 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0cd2315335d94d80b999f857db532613, entries=1, sequenceid=97, filesize=4.9 K
2024-12-03T01:13:46,478 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.08 KB/41039, heapSize ~49.23 KB/50408, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=97, compaction requested=false
2024-12-03T01:13:46,479 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:46,479 DEBUG [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:13:46,479 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/MasterData/WALs/1a7b1a4db65e,35881,1733188371902
2024-12-03T01:13:46,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36469 is added to blk_1073741863_1046 (size=757)
2024-12-03T01:13:46,481 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741863_1046 (size=757)
2024-12-03T01:13:46,482 INFO  [M:0;1a7b1a4db65e:35881 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:13:46,482 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:13:46,482 INFO  [M:0;1a7b1a4db65e:35881 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:35881
2024-12-03T01:13:46,485 DEBUG [M:0;1a7b1a4db65e:35881 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,35881,1733188371902 already deleted, retry=false
2024-12-03T01:13:46,502 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:13:46,504 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,519 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,519 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:46,587 INFO  [M:0;1a7b1a4db65e:35881 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,35881,1733188371902; zookeeper connection closed.
2024-12-03T01:13:46,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:46,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35881-0x10175e1b01f0000, quorum=127.0.0.1:58035, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:13:46,590 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59dcb886{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:46,591 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3dcf703a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:46,591 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:46,591 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39d43134{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:46,591 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@356979bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:46,592 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:46,592 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:46,593 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid 18bf39d1-b6d3-4ecb-81b3-e73d2ab4dd36) service to localhost/127.0.0.1:43249
2024-12-03T01:13:46,593 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:46,593 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@516590ed {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38263,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:37849 , LocalHost:localPort 1a7b1a4db65e/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-03T01:13:46,593 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@516590ed {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:36469,null,null])
java.io.IOException: No block pool offer service for bpid=BP-845475697-172.17.0.2-1733188370852
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:13:46,594 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data4/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:46,594 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@516590ed {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:38263,null,null], DatanodeInfoWithStorage[127.0.0.1:36469,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-845475697-172.17.0.2-1733188370852:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:38263,null,null], DatanodeInfoWithStorage[127.0.0.1:36469,null,null]]
2024-12-03T01:13:46,594 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:46,596 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data3/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:46,597 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49dbde5b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:46,597 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10289dc3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:46,597 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:46,597 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a5b3132{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:46,598 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a8a53d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:46,599 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:13:46,599 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:13:46,599 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:13:46,599 WARN  [BP-845475697-172.17.0.2-1733188370852 heartbeating to localhost/127.0.0.1:43249 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-845475697-172.17.0.2-1733188370852 (Datanode Uuid ed9489eb-e2bf-41cf-8b29-036e4f6c150b) service to localhost/127.0.0.1:43249
2024-12-03T01:13:46,599 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data5/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:46,600 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/cluster_3157865f-f51f-e621-b9a4-f1a41aa4e6de/dfs/data/data6/current/BP-845475697-172.17.0.2-1733188370852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:13:46,600 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:13:46,606 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@69d3d453{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:13:46,607 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@52ca9eab{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:13:46,607 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:13:46,607 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1101772a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:13:46,608 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39523ff9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir/,STOPPED}
2024-12-03T01:13:46,617 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:13:46,646 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:13:46,653 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=87 (was 62)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:43249
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43249
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:43249
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-11-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:43249 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43249
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:43249 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RS-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$$Lambda$796/0x00007f9938b86db0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:43249 from jenkins.hfs.1
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-10-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Abort regionserver monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RS-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-11-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-11-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-10-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-10-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=428 (was 411) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=399 (was 571), ProcessCount=11 (was 11), AvailableMemoryMB=2622 (was 2934)
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=87, OpenFileDescriptor=428, MaxFileDescriptor=1048576, SystemLoadAverage=399, ProcessCount=11, AvailableMemoryMB=2622
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.log.dir so I do NOT create it in target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c4e4114f-bd5c-bffc-652e-758812a37c8c/hadoop.tmp.dir so I do NOT create it in target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616, deleteOnExit=true
2024-12-03T01:13:46,660 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/test.cache.data in system properties and HBase conf
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:13:46,661 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:13:46,661 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:13:46,662 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:13:46,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:13:46,678 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:13:46,772 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:46,777 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:46,780 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:46,780 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:46,780 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:13:46,782 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:46,783 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@117b2140{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:46,784 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3082a870{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:46,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:46,951 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@60016915{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-35127-hadoop-hdfs-3_4_1-tests_jar-_-any-3069292200745614981/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:13:46,952 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@72060fdc{HTTP/1.1, (http/1.1)}{localhost:35127}
2024-12-03T01:13:46,952 INFO  [Time-limited test {}] server.Server(415): Started @174952ms
2024-12-03T01:13:46,972 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:13:47,068 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:47,077 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:47,078 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:47,079 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:47,079 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:13:47,079 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37ebb3d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:47,080 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a39b1ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:47,198 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e38b9fb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-42303-hadoop-hdfs-3_4_1-tests_jar-_-any-9633240965089982252/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:47,199 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b0b4f60{HTTP/1.1, (http/1.1)}{localhost:42303}
2024-12-03T01:13:47,199 INFO  [Time-limited test {}] server.Server(415): Started @175198ms
2024-12-03T01:13:47,200 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:47,247 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:13:47,250 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:13:47,251 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:13:47,251 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:13:47,251 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:13:47,252 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@577a63cd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:13:47,253 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@708ba1ca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:13:47,328 WARN  [Thread-933 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data1/current/BP-53554267-172.17.0.2-1733188426698/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:47,328 WARN  [Thread-934 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data2/current/BP-53554267-172.17.0.2-1733188426698/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:47,345 WARN  [Thread-912 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:47,348 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3336aba147c5818 with lease ID 0xa886c740158c544d: Processing first storage report for DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc from datanode DatanodeRegistration(127.0.0.1:34677, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=40017, infoSecurePort=0, ipcPort=44755, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698)
2024-12-03T01:13:47,348 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3336aba147c5818 with lease ID 0xa886c740158c544d: from storage DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc node DatanodeRegistration(127.0.0.1:34677, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=40017, infoSecurePort=0, ipcPort=44755, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:13:47,348 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa3336aba147c5818 with lease ID 0xa886c740158c544d: Processing first storage report for DS-b9a76415-2495-4430-8c20-120d7fc2bbbe from datanode DatanodeRegistration(127.0.0.1:34677, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=40017, infoSecurePort=0, ipcPort=44755, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698)
2024-12-03T01:13:47,348 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa3336aba147c5818 with lease ID 0xa886c740158c544d: from storage DS-b9a76415-2495-4430-8c20-120d7fc2bbbe node DatanodeRegistration(127.0.0.1:34677, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=40017, infoSecurePort=0, ipcPort=44755, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:47,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e862ae5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-45679-hadoop-hdfs-3_4_1-tests_jar-_-any-11917248352741281992/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:13:47,375 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@471c148c{HTTP/1.1, (http/1.1)}{localhost:45679}
2024-12-03T01:13:47,375 INFO  [Time-limited test {}] server.Server(415): Started @175374ms
2024-12-03T01:13:47,376 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:13:47,493 WARN  [Thread-960 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data4/current/BP-53554267-172.17.0.2-1733188426698/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:47,493 WARN  [Thread-959 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data3/current/BP-53554267-172.17.0.2-1733188426698/current, will proceed with Du for space computation calculation, 
2024-12-03T01:13:47,510 WARN  [Thread-948 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:13:47,514 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb5fd8884099d6746 with lease ID 0xa886c740158c544e: Processing first storage report for DS-c81adf88-3d2a-4014-8378-e2fd756a858d from datanode DatanodeRegistration(127.0.0.1:36813, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=45475, infoSecurePort=0, ipcPort=33909, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698)
2024-12-03T01:13:47,514 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5fd8884099d6746 with lease ID 0xa886c740158c544e: from storage DS-c81adf88-3d2a-4014-8378-e2fd756a858d node DatanodeRegistration(127.0.0.1:36813, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=45475, infoSecurePort=0, ipcPort=33909, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:47,515 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb5fd8884099d6746 with lease ID 0xa886c740158c544e: Processing first storage report for DS-fd3b945b-964e-441b-b8d9-9a4fec14346f from datanode DatanodeRegistration(127.0.0.1:36813, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=45475, infoSecurePort=0, ipcPort=33909, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698)
2024-12-03T01:13:47,515 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5fd8884099d6746 with lease ID 0xa886c740158c544e: from storage DS-fd3b945b-964e-441b-b8d9-9a4fec14346f node DatanodeRegistration(127.0.0.1:36813, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=45475, infoSecurePort=0, ipcPort=33909, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:13:47,600 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2
2024-12-03T01:13:47,609 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/zookeeper_0, clientPort=61778, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:13:47,610 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=61778
2024-12-03T01:13:47,610 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,612 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:13:47,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:13:47,630 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584 with version=8
2024-12-03T01:13:47,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:13:47,632 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:13:47,633 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:13:47,637 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38585
2024-12-03T01:13:47,637 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,638 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,641 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38585 connecting to ZooKeeper ensemble=127.0.0.1:61778
2024-12-03T01:13:47,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385850x0, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:13:47,652 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38585-0x10175e289e50000 connected
2024-12-03T01:13:47,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:13:47,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:13:47,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:13:47,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38585
2024-12-03T01:13:47,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38585
2024-12-03T01:13:47,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38585
2024-12-03T01:13:47,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38585
2024-12-03T01:13:47,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38585
2024-12-03T01:13:47,686 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584, hbase.cluster.distributed=false
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:13:47,711 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:13:47,712 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:13:47,712 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:13:47,713 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43459
2024-12-03T01:13:47,713 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:13:47,714 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:13:47,714 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,716 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,719 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:43459 connecting to ZooKeeper ensemble=127.0.0.1:61778
2024-12-03T01:13:47,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434590x0, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:13:47,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43459-0x10175e289e50001 connected
2024-12-03T01:13:47,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:13:47,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:13:47,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:13:47,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43459
2024-12-03T01:13:47,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43459
2024-12-03T01:13:47,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43459
2024-12-03T01:13:47,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43459
2024-12-03T01:13:47,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43459
2024-12-03T01:13:47,725 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:47,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:13:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:13:47,729 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:47,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:13:47,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:13:47,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:47,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:47,733 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:13:47,734 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,38585,1733188427632 from backup master directory
2024-12-03T01:13:47,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:13:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:47,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:13:47,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:13:47,736 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:13:47,736 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:47,744 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:38585
2024-12-03T01:13:47,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:13:47,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:13:47,750 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/hbase.id with ID: e1a19a19-8139-48e5-a138-2a586cafcb58
2024-12-03T01:13:47,764 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:47,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:47,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:47,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:13:47,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:13:47,780 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:13:47,781 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:13:47,781 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:13:47,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:13:47,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:13:47,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:48,199 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store
2024-12-03T01:13:48,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:13:48,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:13:48,208 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:48,208 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:13:48,208 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:13:48,209 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/.initializing
2024-12-03T01:13:48,209 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:48,212 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C38585%2C1733188427632, suffix=, logDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632, archiveDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/oldWALs, maxLogs=10
2024-12-03T01:13:48,212 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C38585%2C1733188427632.1733188428212
2024-12-03T01:13:48,221 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212
2024-12-03T01:13:48,221 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45475:45475),(127.0.0.1/127.0.0.1:40017:40017)]
2024-12-03T01:13:48,222 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:13:48,222 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:48,222 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:13:48,222 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:13:48,223 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:13:48,225 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:13:48,225 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,225 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,225 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:13:48,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:13:48,227 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:13:48,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:13:48,229 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:13:48,229 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,229 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:13:48,229 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:13:48,231 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:13:48,231 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,231 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:13:48,232 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:13:48,232 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:13:48,234 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:13:48,235 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:13:48,237 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:13:48,238 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=757674, jitterRate=-0.036567896604537964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:13:48,239 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:13:48,239 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:13:48,243 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e9b753a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:13:48,244 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:13:48,244 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:13:48,244 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:13:48,244 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:13:48,245 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T01:13:48,245 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-03T01:13:48,245 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:13:48,247 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:13:48,248 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:13:48,255 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:13:48,256 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:13:48,256 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:13:48,258 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:13:48,258 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:13:48,259 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:13:48,260 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:13:48,261 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:13:48,262 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:13:48,264 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:13:48,266 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:13:48,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:13:48,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:13:48,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,268 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,38585,1733188427632, sessionid=0x10175e289e50000, setting cluster-up flag (Was=false)
2024-12-03T01:13:48,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,277 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:13:48,278 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:48,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,303 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:13:48,304 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:48,325 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:13:48,325 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:13:48,325 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,38585,1733188427632
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,326 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:13:48,327 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,329 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188458329
2024-12-03T01:13:48,329 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:13:48,330 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,330 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:13:48,331 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:13:48,331 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:13:48,331 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:13:48,331 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,331 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:13:48,331 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:13:48,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-12-03T01:13:48,332 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188428331,5,FailOnTimeoutGroup]
2024-12-03T01:13:48,333 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188428332,5,FailOnTimeoutGroup]
2024-12-03T01:13:48,333 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,333 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:13:48,333 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,333 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,341 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:43459
2024-12-03T01:13:48,343 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1008): ClusterId : e1a19a19-8139-48e5-a138-2a586cafcb58
2024-12-03T01:13:48,343 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:13:48,347 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:13:48,347 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:13:48,349 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:13:48,349 DEBUG [RS:0;1a7b1a4db65e:43459 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1158fcf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:13:48,350 DEBUG [RS:0;1a7b1a4db65e:43459 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f01ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:13:48,350 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:13:48,350 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:13:48,350 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:13:48,351 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,38585,1733188427632 with isa=1a7b1a4db65e/172.17.0.2:43459, startcode=1733188427710
2024-12-03T01:13:48,351 DEBUG [RS:0;1a7b1a4db65e:43459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:13:48,353 INFO  [RS-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49197, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:13:48,354 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38585 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,354 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38585 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,363 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584
2024-12-03T01:13:48,363 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:40095
2024-12-03T01:13:48,363 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:13:48,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:13:48,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:13:48,365 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:13:48,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:13:48,366 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584
2024-12-03T01:13:48,366 DEBUG [RS:0;1a7b1a4db65e:43459 {}] zookeeper.ZKUtil(111): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,366 WARN  [RS:0;1a7b1a4db65e:43459 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:13:48,366 INFO  [RS:0;1a7b1a4db65e:43459 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:13:48,367 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,367 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,43459,1733188427710]
2024-12-03T01:13:48,375 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:13:48,375 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:13:48,376 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:13:48,377 INFO  [RS:0;1a7b1a4db65e:43459 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:13:48,377 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,377 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:13:48,378 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,378 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,379 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,379 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,379 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:13:48,379 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:13:48,379 DEBUG [RS:0;1a7b1a4db65e:43459 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:13:48,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:13:48,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:13:48,380 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,380 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,380 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,380 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,380 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,43459,1733188427710-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:13:48,395 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:13:48,396 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,43459,1733188427710-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:48,402 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:13:48,405 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:13:48,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:13:48,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:13:48,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:13:48,410 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:13:48,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,411 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/1588230740
2024-12-03T01:13:48,412 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.Replication(204): 1a7b1a4db65e,43459,1733188427710 started
2024-12-03T01:13:48,412 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,43459,1733188427710, RpcServer on 1a7b1a4db65e/172.17.0.2:43459, sessionid=0x10175e289e50001
2024-12-03T01:13:48,412 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/1588230740
2024-12-03T01:13:48,412 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:13:48,412 DEBUG [RS:0;1a7b1a4db65e:43459 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,412 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,43459,1733188427710'
2024-12-03T01:13:48,412 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:13:48,412 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:13:48,413 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:13:48,413 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:13:48,413 DEBUG [RS:0;1a7b1a4db65e:43459 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,413 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,43459,1733188427710'
2024-12-03T01:13:48,413 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:13:48,414 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:13:48,415 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:13:48,415 DEBUG [RS:0;1a7b1a4db65e:43459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:13:48,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:13:48,415 INFO  [RS:0;1a7b1a4db65e:43459 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:13:48,415 INFO  [RS:0;1a7b1a4db65e:43459 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:13:48,418 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:13:48,419 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814221, jitterRate=0.03533615171909332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:13:48,419 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:13:48,419 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:13:48,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:13:48,421 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:13:48,421 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:13:48,421 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:13:48,422 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:13:48,423 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:13:48,518 INFO  [RS:0;1a7b1a4db65e:43459 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C43459%2C1733188427710, suffix=, logDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710, archiveDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/oldWALs, maxLogs=32
2024-12-03T01:13:48,519 INFO  [RS:0;1a7b1a4db65e:43459 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:13:48,526 INFO  [RS:0;1a7b1a4db65e:43459 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:13:48,526 DEBUG [RS:0;1a7b1a4db65e:43459 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45475:45475),(127.0.0.1/127.0.0.1:40017:40017)]
2024-12-03T01:13:48,574 DEBUG [1a7b1a4db65e:38585 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:13:48,574 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,577 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,43459,1733188427710, state=OPENING
2024-12-03T01:13:48,578 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:13:48,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:48,583 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,43459,1733188427710}]
2024-12-03T01:13:48,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:13:48,583 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:13:48,737 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,737 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:13:48,739 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46268, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:13:48,743 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:13:48,743 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:13:48,745 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C43459%2C1733188427710.meta, suffix=.meta, logDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710, archiveDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/oldWALs, maxLogs=32
2024-12-03T01:13:48,746 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta
2024-12-03T01:13:48,757 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta
2024-12-03T01:13:48,757 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45475:45475),(127.0.0.1/127.0.0.1:40017:40017)]
2024-12-03T01:13:48,757 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:13:48,757 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:13:48,757 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:13:48,757 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:13:48,757 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:13:48,758 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:48,758 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:13:48,758 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:13:48,759 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:13:48,760 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:13:48,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,761 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,761 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:13:48,762 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:13:48,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:13:48,764 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:13:48,764 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,764 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:13:48,765 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/1588230740
2024-12-03T01:13:48,766 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/meta/1588230740
2024-12-03T01:13:48,768 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:13:48,770 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:13:48,771 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739574, jitterRate=-0.05958370864391327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:13:48,771 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:13:48,772 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188428736
2024-12-03T01:13:48,774 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:13:48,774 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:13:48,776 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,777 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,43459,1733188427710, state=OPEN
2024-12-03T01:13:48,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:13:48,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:13:48,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:13:48,783 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:13:48,786 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:13:48,786 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,43459,1733188427710 in 200 msec
2024-12-03T01:13:48,788 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:13:48,788 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 365 msec
2024-12-03T01:13:48,790 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 480 msec
2024-12-03T01:13:48,790 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188428790, completionTime=-1
2024-12-03T01:13:48,790 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:13:48,790 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:13:48,791 DEBUG [hconnection-0x2052f0e0-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:13:48,792 INFO  [RS-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46270, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:13:48,794 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:13:48,794 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188488794
2024-12-03T01:13:48,794 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188548794
2024-12-03T01:13:48,794 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-03T01:13:48,799 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,799 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,799 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,800 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:38585, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,800 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:48,800 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:13:48,800 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:13:48,801 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:13:48,802 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:13:48,804 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:13:48,804 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:48,805 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:13:48,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:13:48,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:13:48,819 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => f733499e9a7b158639ffa297effcb71d, NAME => 'hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584
2024-12-03T01:13:48,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:13:48,827 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing f733499e9a7b158639ffa297effcb71d, disabling compactions & flushes
2024-12-03T01:13:48,827 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d. after waiting 0 ms
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:48,827 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:48,827 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for f733499e9a7b158639ffa297effcb71d:

2024-12-03T01:13:48,829 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:13:48,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188428829"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188428829"}]},"ts":"1733188428829"}
2024-12-03T01:13:48,831 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:13:48,832 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:13:48,832 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188428832"}]},"ts":"1733188428832"}
2024-12-03T01:13:48,834 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:13:48,839 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f733499e9a7b158639ffa297effcb71d, ASSIGN}]
2024-12-03T01:13:48,840 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f733499e9a7b158639ffa297effcb71d, ASSIGN
2024-12-03T01:13:48,842 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=f733499e9a7b158639ffa297effcb71d, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,43459,1733188427710; forceNewPlan=false, retain=false
2024-12-03T01:13:48,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:48,992 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f733499e9a7b158639ffa297effcb71d, regionState=OPENING, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:48,995 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure f733499e9a7b158639ffa297effcb71d, server=1a7b1a4db65e,43459,1733188427710}]
2024-12-03T01:13:49,148 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:49,152 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:49,152 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => f733499e9a7b158639ffa297effcb71d, NAME => 'hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:13:49,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:49,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,153 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,156 INFO  [StoreOpener-f733499e9a7b158639ffa297effcb71d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f733499e9a7b158639ffa297effcb71d 
2024-12-03T01:13:49,158 INFO  [StoreOpener-f733499e9a7b158639ffa297effcb71d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f733499e9a7b158639ffa297effcb71d columnFamilyName info
2024-12-03T01:13:49,158 DEBUG [StoreOpener-f733499e9a7b158639ffa297effcb71d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:49,158 INFO  [StoreOpener-f733499e9a7b158639ffa297effcb71d-1 {}] regionserver.HStore(327): Store=f733499e9a7b158639ffa297effcb71d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:13:49,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/namespace/f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/namespace/f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,161 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for f733499e9a7b158639ffa297effcb71d
2024-12-03T01:13:49,164 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/hbase/namespace/f733499e9a7b158639ffa297effcb71d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:13:49,165 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened f733499e9a7b158639ffa297effcb71d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=766341, jitterRate=-0.025547057390213013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:13:49,166 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for f733499e9a7b158639ffa297effcb71d:

2024-12-03T01:13:49,167 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d., pid=6, masterSystemTime=1733188429148
2024-12-03T01:13:49,169 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:49,169 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:13:49,170 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f733499e9a7b158639ffa297effcb71d, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:49,175 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:13:49,175 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure f733499e9a7b158639ffa297effcb71d, server=1a7b1a4db65e,43459,1733188427710 in 177 msec
2024-12-03T01:13:49,177 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:13:49,177 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=f733499e9a7b158639ffa297effcb71d, ASSIGN in 336 msec
2024-12-03T01:13:49,178 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:13:49,178 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188429178"}]},"ts":"1733188429178"}
2024-12-03T01:13:49,180 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:13:49,183 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:13:49,185 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 383 msec
2024-12-03T01:13:49,203 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:13:49,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:13:49,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:49,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:13:49,209 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:13:49,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:13:49,224 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 14 msec
2024-12-03T01:13:49,232 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:13:49,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:13:49,245 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 12 msec
2024-12-03T01:13:49,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:13:49,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.525sec
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:13:49,261 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:13:49,266 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:13:49,266 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:13:49,266 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38585,1733188427632-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:13:49,328 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dfbd142 to 127.0.0.1:61778 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42eee7fc
2024-12-03T01:13:49,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52be4020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:13:49,334 DEBUG [hconnection-0x67da9c8c-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:13:49,336 INFO  [RS-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46272, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:13:49,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,38585,1733188427632
2024-12-03T01:13:49,338 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:13:49,342 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:13:49,342 INFO  [Time-limited test {}] wal.TestLogRolling(297): Starting testLogRollOnPipelineRestart
2024-12-03T01:13:49,342 INFO  [Time-limited test {}] wal.TestLogRolling(300): Replication=2
2024-12-03T01:13:49,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T01:13:49,345 INFO  [RS-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50870, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T01:13:49,346 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T01:13:49,346 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T01:13:49,346 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:13:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T01:13:49,349 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:13:49,349 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:49,349 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 9
2024-12-03T01:13:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:13:49,351 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:13:49,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741837_1013 (size=395)
2024-12-03T01:13:49,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741837_1013 (size=395)
2024-12-03T01:13:49,364 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 836c2d63d714401836909ae12c7d605d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584
2024-12-03T01:13:49,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34677 is added to blk_1073741838_1014 (size=78)
2024-12-03T01:13:49,374 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36813 is added to blk_1073741838_1014 (size=78)
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1681): Closing 836c2d63d714401836909ae12c7d605d, disabling compactions & flushes
2024-12-03T01:13:49,375 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d. after waiting 0 ms
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,375 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,375 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1635): Region close journal for 836c2d63d714401836909ae12c7d605d:

2024-12-03T01:13:49,377 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:13:49,377 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733188429377"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188429377"}]},"ts":"1733188429377"}
2024-12-03T01:13:49,379 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:13:49,380 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:13:49,380 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188429380"}]},"ts":"1733188429380"}
2024-12-03T01:13:49,382 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-03T01:13:49,387 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=836c2d63d714401836909ae12c7d605d, ASSIGN}]
2024-12-03T01:13:49,388 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=836c2d63d714401836909ae12c7d605d, ASSIGN
2024-12-03T01:13:49,389 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=836c2d63d714401836909ae12c7d605d, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,43459,1733188427710; forceNewPlan=false, retain=false
2024-12-03T01:13:49,540 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=836c2d63d714401836909ae12c7d605d, regionState=OPENING, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:49,543 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 836c2d63d714401836909ae12c7d605d, server=1a7b1a4db65e,43459,1733188427710}]
2024-12-03T01:13:49,696 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:49,701 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,702 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 836c2d63d714401836909ae12c7d605d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:13:49,702 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,702 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:13:49,702 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,703 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,704 INFO  [StoreOpener-836c2d63d714401836909ae12c7d605d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 836c2d63d714401836909ae12c7d605d 
2024-12-03T01:13:49,710 INFO  [StoreOpener-836c2d63d714401836909ae12c7d605d-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 836c2d63d714401836909ae12c7d605d columnFamilyName info
2024-12-03T01:13:49,711 DEBUG [StoreOpener-836c2d63d714401836909ae12c7d605d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:13:49,711 INFO  [StoreOpener-836c2d63d714401836909ae12c7d605d-1 {}] regionserver.HStore(327): Store=836c2d63d714401836909ae12c7d605d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:13:49,712 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,713 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,716 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 836c2d63d714401836909ae12c7d605d
2024-12-03T01:13:49,721 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:13:49,727 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 836c2d63d714401836909ae12c7d605d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719613, jitterRate=-0.08496509492397308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:13:49,728 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 836c2d63d714401836909ae12c7d605d:

2024-12-03T01:13:49,744 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d., pid=11, masterSystemTime=1733188429696
2024-12-03T01:13:49,753 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,753 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:49,754 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=836c2d63d714401836909ae12c7d605d, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,43459,1733188427710
2024-12-03T01:13:49,760 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-03T01:13:49,761 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 836c2d63d714401836909ae12c7d605d, server=1a7b1a4db65e,43459,1733188427710 in 213 msec
2024-12-03T01:13:49,769 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T01:13:49,769 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=836c2d63d714401836909ae12c7d605d, ASSIGN in 373 msec
2024-12-03T01:13:49,770 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:13:49,770 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188429770"}]},"ts":"1733188429770"}
2024-12-03T01:13:49,774 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-03T01:13:49,778 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:13:49,780 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 433 msec
2024-12-03T01:13:49,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:50,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:51,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:52,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:53,834 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:13:53,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:13:53,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:54,375 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-03T01:13:54,375 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-03T01:13:54,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:55,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:56,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:57,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:58,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace
2024-12-03T01:13:58,331 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer
2024-12-03T01:13:58,332 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T01:13:58,333 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-03T01:13:58,959 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:13:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38585 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:13:59,352 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart, procId: 9 completed
2024-12-03T01:13:59,355 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T01:13:59,355 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:13:59,960 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:00,961 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:01,361 INFO  [Time-limited test {}] wal.TestLogRolling(337): log.getCurrentFileName()): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:01,362 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,362 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,362 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK], DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]) is bad.
2024-12-03T01:14:01,362 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK], DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]) is bad.
2024-12-03T01:14:01,362 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,363 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK], DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36813,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]) is bad.
2024-12-03T01:14:01,363 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:51876 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36813:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51876 dst: /127.0.0.1:36813
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,364 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:46878 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46878 dst: /127.0.0.1:34677
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,364 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-648233880_22 at /127.0.0.1:46846 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46846 dst: /127.0.0.1:34677
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,364 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:46868 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46868 dst: /127.0.0.1:34677
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,363 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:51866 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36813:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51866 dst: /127.0.0.1:36813
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,364 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-648233880_22 at /127.0.0.1:51824 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36813:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51824 dst: /127.0.0.1:36813
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,366 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e862ae5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:01,366 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@471c148c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:01,366 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:01,367 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@708ba1ca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:01,367 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@577a63cd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:01,369 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:01,369 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:01,372 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:01,372 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid 84fe90b5-7615-481c-b128-1de749075c71) service to localhost/127.0.0.1:40095
2024-12-03T01:14:01,373 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data4/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:01,374 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:01,376 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data3/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:01,387 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:01,391 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:01,391 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:01,391 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:01,391 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:01,392 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@631bcbaf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:01,392 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d1b8d70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:01,511 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@545a7404{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-34967-hadoop-hdfs-3_4_1-tests_jar-_-any-7695079036451389987/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:01,512 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@31be664c{HTTP/1.1, (http/1.1)}{localhost:34967}
2024-12-03T01:14:01,512 INFO  [Time-limited test {}] server.Server(415): Started @189511ms
2024-12-03T01:14:01,513 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:01,541 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,541 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1017 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1017
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,541 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:01,541 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-648233880_22 at /127.0.0.1:37966 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37966 dst: /127.0.0.1:34677
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,542 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:37978 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37978 dst: /127.0.0.1:34677
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,542 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:37994 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34677:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37994 dst: /127.0.0.1:34677
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:01,546 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e38b9fb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:01,547 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b0b4f60{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:01,547 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:01,547 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a39b1ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:01,547 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37ebb3d0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:01,553 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:01,553 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:01,553 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid c191c7dc-e61b-464d-8628-7cdf6dce854a) service to localhost/127.0.0.1:40095
2024-12-03T01:14:01,553 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:01,554 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data1/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:01,554 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data2/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:01,554 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:01,566 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:01,572 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:01,574 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:01,574 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:01,574 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:14:01,575 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@500f8b9e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:01,575 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6203c463{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:01,619 WARN  [Thread-1094 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:01,622 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8058d6d20803166d with lease ID 0xa886c740158c544f: from storage DS-c81adf88-3d2a-4014-8378-e2fd756a858d node DatanodeRegistration(127.0.0.1:46693, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=35575, infoSecurePort=0, ipcPort=36355, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:01,622 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8058d6d20803166d with lease ID 0xa886c740158c544f: from storage DS-fd3b945b-964e-441b-b8d9-9a4fec14346f node DatanodeRegistration(127.0.0.1:46693, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=35575, infoSecurePort=0, ipcPort=36355, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:01,696 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7650d9c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-46277-hadoop-hdfs-3_4_1-tests_jar-_-any-11170340104356396087/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:01,697 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3deee880{HTTP/1.1, (http/1.1)}{localhost:46277}
2024-12-03T01:14:01,697 INFO  [Time-limited test {}] server.Server(415): Started @189697ms
2024-12-03T01:14:01,699 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:01,807 WARN  [Thread-1125 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:01,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe114c879e04a575 with lease ID 0xa886c740158c5450: from storage DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc node DatanodeRegistration(127.0.0.1:36135, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=35851, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:14:01,810 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe114c879e04a575 with lease ID 0xa886c740158c5450: from storage DS-b9a76415-2495-4430-8c20-120d7fc2bbbe node DatanodeRegistration(127.0.0.1:36135, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=35851, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:01,962 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:02,721 INFO  [Time-limited test {}] wal.TestLogRolling(349): Data Nodes restarted
2024-12-03T01:14:02,723 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-03T01:14:02,724 WARN  [RS:0;1a7b1a4db65e:43459.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=5, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:02,724 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C43459%2C1733188427710:(num 1733188428519) roll requested
2024-12-03T01:14:02,725 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:02,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43459 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43459 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46272 deadline: 1733188452723, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-03T01:14:02,731 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 newFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:02,731 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=5, requesting roll of WAL
2024-12-03T01:14:02,731 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 with entries=5, filesize=2.09 KB; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:02,732 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35575:35575),(127.0.0.1/127.0.0.1:35851:35851)]
2024-12-03T01:14:02,732 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 is not closed yet, will try archiving it next time
2024-12-03T01:14:02,732 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:02,732 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:02,732 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:02,732 WARN  [IPC Server handler 2 on default port 40095 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1015
2024-12-03T01:14:02,733 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 after 1ms
2024-12-03T01:14:02,963 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:03,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:04,964 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:05,621 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1015: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T01:14:05,965 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:06,734 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 after 4001ms
2024-12-03T01:14:06,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:07,966 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:08,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:09,967 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:10,968 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:11,969 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:12,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:13,970 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:14,791 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-03T01:14:14,971 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:15,972 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:16,794 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018
java.io.IOException: Bad response ERROR for BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018 from datanode DatanodeInfoWithStorage[127.0.0.1:36135,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:16,795 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46693,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK], DatanodeInfoWithStorage[127.0.0.1:36135,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36135,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]) is bad.
2024-12-03T01:14:16,795 WARN  [PacketResponder: BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36135] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:16,795 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:46550 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:46693:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46550 dst: /127.0.0.1:46693
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:16,795 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:50892 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:36135:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50892 dst: /127.0.0.1:36135
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:16,797 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7650d9c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:16,797 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3deee880{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:16,797 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:16,798 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6203c463{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:16,798 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@500f8b9e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:16,801 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:16,801 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:16,801 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:16,801 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid c191c7dc-e61b-464d-8628-7cdf6dce854a) service to localhost/127.0.0.1:40095
2024-12-03T01:14:16,802 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data1/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:16,802 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data2/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:16,802 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:16,813 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:16,816 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:16,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:16,817 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:16,817 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:16,817 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d7a54ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:16,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52286d38{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:16,939 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@532f8be9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-37521-hadoop-hdfs-3_4_1-tests_jar-_-any-3073672136549706162/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:16,940 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1bc459ac{HTTP/1.1, (http/1.1)}{localhost:37521}
2024-12-03T01:14:16,940 INFO  [Time-limited test {}] server.Server(415): Started @204939ms
2024-12-03T01:14:16,941 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:16,959 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:16,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1715205017_22 at /127.0.0.1:60382 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741839_1018] {}] datanode.DataXceiver(331): 127.0.0.1:46693:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60382 dst: /127.0.0.1:46693
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:16,963 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@545a7404{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:16,964 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@31be664c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:16,964 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:16,964 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d1b8d70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:16,964 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@631bcbaf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:16,967 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:16,967 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid 84fe90b5-7615-481c-b128-1de749075c71) service to localhost/127.0.0.1:40095
2024-12-03T01:14:16,967 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:16,967 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:16,968 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data3/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:16,968 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data4/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:16,969 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:16,973 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:16,987 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:16,991 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:16,997 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:16,997 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:16,997 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:16,998 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1db1a42c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:16,998 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@280570d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:17,069 WARN  [Thread-1169 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:17,072 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb63007e4801987ac with lease ID 0xa886c740158c5451: from storage DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc node DatanodeRegistration(127.0.0.1:35755, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=45407, infoSecurePort=0, ipcPort=33649, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:17,072 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb63007e4801987ac with lease ID 0xa886c740158c5451: from storage DS-b9a76415-2495-4430-8c20-120d7fc2bbbe node DatanodeRegistration(127.0.0.1:35755, datanodeUuid=c191c7dc-e61b-464d-8628-7cdf6dce854a, infoPort=45407, infoSecurePort=0, ipcPort=33649, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:17,124 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@129adbbb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/java.io.tmpdir/jetty-localhost-41267-hadoop-hdfs-3_4_1-tests_jar-_-any-10978621101062332670/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:17,124 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71144b43{HTTP/1.1, (http/1.1)}{localhost:41267}
2024-12-03T01:14:17,124 INFO  [Time-limited test {}] server.Server(415): Started @205124ms
2024-12-03T01:14:17,126 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:17,233 WARN  [Thread-1200 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:17,236 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x182c026c502c8d0b with lease ID 0xa886c740158c5452: from storage DS-c81adf88-3d2a-4014-8378-e2fd756a858d node DatanodeRegistration(127.0.0.1:33419, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=41073, infoSecurePort=0, ipcPort=37229, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 8, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:17,236 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x182c026c502c8d0b with lease ID 0xa886c740158c5452: from storage DS-fd3b945b-964e-441b-b8d9-9a4fec14346f node DatanodeRegistration(127.0.0.1:33419, datanodeUuid=84fe90b5-7615-481c-b128-1de749075c71, infoPort=41073, infoSecurePort=0, ipcPort=37229, storageInfo=lv=-57;cid=testClusterID;nsid=1963370660;c=1733188426698), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:17,599 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:14:17,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:18,143 INFO  [Time-limited test {}] wal.TestLogRolling(366): Data Nodes restarted
2024-12-03T01:14:18,145 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-03T01:14:18,146 WARN  [RS:0;1a7b1a4db65e:43459.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=8, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46693,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,146 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C43459%2C1733188427710:(num 1733188442724) roll requested
2024-12-03T01:14:18,147 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43459 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46693,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43459 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46272 deadline: 1733188468146, exception=org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-03T01:14:18,154 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 newFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:18,154 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=8, requesting roll of WAL
2024-12-03T01:14:18,154 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:18,155 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41073:41073),(127.0.0.1/127.0.0.1:45407:45407)]
2024-12-03T01:14:18,155 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 is not closed yet, will try archiving it next time
2024-12-03T01:14:18,155 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46693,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,155 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46693,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,155 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:18,155 WARN  [IPC Server handler 2 on default port 40095 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1020
2024-12-03T01:14:18,156 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 after 1ms
2024-12-03T01:14:18,329 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=95, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,330 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C38585%2C1733188427632:(num 1733188428212) roll requested
2024-12-03T01:14:18,330 ERROR [ProcExecTimeout {}] region.RegionProcedureStore(422): Failed to delete pids=[4, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,330 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C38585%2C1733188427632.1733188458330
2024-12-03T01:14:18,330 ERROR [ProcExecTimeout {}] procedure2.TimeoutExecutorThread(124): Ignoring pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner exception: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
java.io.UncheckedIOException: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.delete(RegionProcedureStore.java:423) ~[classes/:?]
	at org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner.periodicExecute(CompletedProcedureCleaner.java:135) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.executeInMemoryChore(TimeoutExecutorThread.java:122) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.execDelayedProcedure(TimeoutExecutorThread.java:101) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:68) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
Caused by: org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,336 WARN  [master:store-WAL-Roller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=95, requesting roll of WAL
2024-12-03T01:14:18,336 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 with entries=92, filesize=46.00 KB; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188458330
2024-12-03T01:14:18,337 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45407:45407),(127.0.0.1/127.0.0.1:41073:41073)]
2024-12-03T01:14:18,337 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,337 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 is not closed yet, will try archiving it next time
2024-12-03T01:14:18,337 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:18,337 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212
2024-12-03T01:14:18,337 WARN  [IPC Server handler 1 on default port 40095 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741830_1017
2024-12-03T01:14:18,337 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 after 0ms
2024-12-03T01:14:18,974 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:19,350 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:14:19,352 INFO  [RS-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56022, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:14:19,975 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:20,073 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741839_1020: GenerationStamp not matched, existing replica is blk_1073741839_1018
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T01:14:20,976 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:21,977 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:22,156 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 after 4001ms
2024-12-03T01:14:22,338 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632/1a7b1a4db65e%2C38585%2C1733188427632.1733188428212 after 4001ms
2024-12-03T01:14:22,978 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:23,236 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 2) replica(s):
0) Failed to delete replica blk_1073741830_1017: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T01:14:23,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:24,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 23930
2024-12-03T01:14:24,979 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:25,980 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:26,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:27,981 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:28,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:29,838 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T01:14:29,838 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T01:14:29,982 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:30,178 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:30,183 DEBUG [Time-limited test {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147 newFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:30,185 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:30,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45407:45407),(127.0.0.1/127.0.0.1:41073:41073)]
2024-12-03T01:14:30,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147 is not closed yet, will try archiving it next time
2024-12-03T01:14:30,185 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:30,185 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:30,185 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519 after 0ms
2024-12-03T01:14:30,185 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:30,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741840_1021 (size=1264)
2024-12-03T01:14:30,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741840_1021 (size=1264)
2024-12-03T01:14:30,193 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733188429166/Put/vlen=162/seqid=0]
2024-12-03T01:14:30,193 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [default/info:d/1733188429215/Put/vlen=9/seqid=0]
2024-12-03T01:14:30,193 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #5: [hbase/info:d/1733188429237/Put/vlen=7/seqid=0]
2024-12-03T01:14:30,194 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733188429728/Put/vlen=218/seqid=0]
2024-12-03T01:14:30,194 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #4: [row1002/info:/1733188439358/Put/vlen=1045/seqid=0]
2024-12-03T01:14:30,194 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188428519
2024-12-03T01:14:30,194 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:30,194 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:30,194 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724 after 0ms
2024-12-03T01:14:30,194 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:30,198 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #6: [row1003/info:/1733188452788/Put/vlen=1045/seqid=0]
2024-12-03T01:14:30,198 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #7: [row1004/info:/1733188454792/Put/vlen=1045/seqid=0]
2024-12-03T01:14:30,198 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188442724
2024-12-03T01:14:30,198 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:30,198 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:30,198 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147 after 0ms
2024-12-03T01:14:30,198 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188458147
2024-12-03T01:14:30,201 DEBUG [Time-limited test {}] wal.TestLogRolling(389): #9: [row1005/info:/1733188468176/Put/vlen=1045/seqid=0]
2024-12-03T01:14:30,201 DEBUG [Time-limited test {}] wal.TestLogRolling(380): recovering lease for hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:30,201 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:30,202 WARN  [IPC Server handler 4 on default port 40095 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741842_1025
2024-12-03T01:14:30,202 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 after 1ms
2024-12-03T01:14:30,983 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:31,073 WARN  [ResponseProcessor for block BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:31,073 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-648233880_22 at /127.0.0.1:33110 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:35755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33110 dst: /127.0.0.1:35755
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35755 remote=/127.0.0.1:33110]. Total timeout mills is 60000, 59111 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:31,073 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-648233880_22 at /127.0.0.1:48616 [Receiving block BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025] {}] datanode.DataXceiver(331): 127.0.0.1:33419:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48616 dst: /127.0.0.1:33419
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:31,073 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 block BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35755,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK], DatanodeInfoWithStorage[127.0.0.1:33419,DS-c81adf88-3d2a-4014-8378-e2fd756a858d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35755,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]) is bad.
2024-12-03T01:14:31,078 WARN  [DataStreamer for file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 block BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:31,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741842_1026 (size=85)
2024-12-03T01:14:31,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741842_1026 (size=85)
2024-12-03T01:14:31,984 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:32,984 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:33,985 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:34,203 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 after 4002ms
2024-12-03T01:14:34,203 DEBUG [Time-limited test {}] wal.TestLogRolling(384): Reading WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:34,208 DEBUG [Time-limited test {}] wal.TestLogRolling(396): EOF reading file /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:34,208 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 836c2d63d714401836909ae12c7d605d 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-03T01:14:34,209 WARN  [RS:0;1a7b1a4db65e:43459.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,209 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C43459%2C1733188427710:(num 1733188470177) roll requested
2024-12-03T01:14:34,209 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 836c2d63d714401836909ae12c7d605d:

2024-12-03T01:14:34,210 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.1733188474209
2024-12-03T01:14:34,210 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,210 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing f733499e9a7b158639ffa297effcb71d 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:14:34,211 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for f733499e9a7b158639ffa297effcb71d:

2024-12-03T01:14:34,211 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,211 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.42 KB
2024-12-03T01:14:34,212 WARN  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0.append-pool-0 {}] wal.FSHLog$RingBufferEventHandler(1189): Append sequenceId=15, requesting roll of WAL
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,212 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-03T01:14:34,212 INFO  [Time-limited test {}] wal.TestLogRolling(416): org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:14:34,218 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-03T01:14:34,218 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3dfbd142 to 127.0.0.1:61778
2024-12-03T01:14:34,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:14:34,218 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:14:34,219 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=795993279, stopped=false
2024-12-03T01:14:34,219 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,38585,1733188427632
2024-12-03T01:14:34,222 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.TestLogRolling$2(324): preLogRoll: oldFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 newFile=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188474209
2024-12-03T01:14:34,222 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=11, requesting roll of WAL
2024-12-03T01:14:34,222 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188474209
2024-12-03T01:14:34,222 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41073:41073),(127.0.0.1/127.0.0.1:45407:45407)]
2024-12-03T01:14:34,222 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 is not closed yet, will try archiving it next time
2024-12-03T01:14:34,222 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(197): WAL FSHLog 1a7b1a4db65e%2C43459%2C1733188427710.meta:.meta(num 1733188428746) roll requested
2024-12-03T01:14:34,222 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188474222.meta
2024-12-03T01:14:34,222 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:14:34,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:34,223 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-53554267-172.17.0.2-1733188426698:blk_1073741842_1025 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,223 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:34,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:14:34,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:34,224 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:14:34,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:14:34,224 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 after 1ms
2024-12-03T01:14:34,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:14:34,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:14:34,225 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177 to hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/oldWALs/1a7b1a4db65e%2C43459%2C1733188427710.1733188470177
2024-12-03T01:14:34,225 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,43459,1733188427710' *****
2024-12-03T01:14:34,225 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:14:34,225 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:14:34,228 INFO  [RS:0;1a7b1a4db65e:43459 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:14:34,228 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3579): Received CLOSE for 836c2d63d714401836909ae12c7d605d
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3579): Received CLOSE for f733499e9a7b158639ffa297effcb71d
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,43459,1733188427710
2024-12-03T01:14:34,229 DEBUG [RS:0;1a7b1a4db65e:43459 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:14:34,229 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 836c2d63d714401836909ae12c7d605d, disabling compactions & flushes
2024-12-03T01:14:34,229 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:14:34,229 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:14:34,229 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:14:34,229 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d. after waiting 0 ms
2024-12-03T01:14:34,229 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:14:34,230 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 836c2d63d714401836909ae12c7d605d 1/1 column families, dataSize=4.20 KB heapSize=4.98 KB
2024-12-03T01:14:34,230 WARN  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-03T01:14:34,233 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-03T01:14:34,233 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1603): Online Regions={836c2d63d714401836909ae12c7d605d=TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d., f733499e9a7b158639ffa297effcb71d=hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d., 1588230740=hbase:meta,,1.1588230740}
2024-12-03T01:14:34,233 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 836c2d63d714401836909ae12c7d605d, f733499e9a7b158639ffa297effcb71d
2024-12-03T01:14:34,233 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:14:34,233 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:14:34,233 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:14:34,233 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:14:34,233 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:14:34,233 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.90 KB heapSize=5.89 KB
2024-12-03T01:14:34,233 WARN  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-03T01:14:34,234 WARN  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultMemStore(92): Snapshot called again without clearing previous. Doing nothing. Another ongoing flush or did we fail last attempt?
2024-12-03T01:14:34,234 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:14:34,234 ERROR [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2808): ***** ABORTING region server 1a7b1a4db65e,43459,1733188427710: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967) ~[classes/:?]
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168) ~[disruptor-3.4.4.jar:?]
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) ~[disruptor-3.4.4.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,234 ERROR [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2815): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint]
2024-12-03T01:14:34,234 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for java.lang:type=Memory
2024-12-03T01:14:34,235 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC
2024-12-03T01:14:34,235 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication
2024-12-03T01:14:34,235 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server
2024-12-03T01:14:34,235 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2819): Dump of metrics as JSON on abort: {
  "beans": [
    {
      "name": "java.lang:type=Memory",
      "modelerType": "sun.management.MemoryImpl",
      "ObjectPendingFinalizationCount": 0,
      "HeapMemoryUsage": {
        "committed": 1048576000,
        "init": 1048576000,
        "max": 2306867200,
        "used": 648635840
      },
      "NonHeapMemoryUsage": {
        "committed": 169738240,
        "init": 7667712,
        "max": -1,
        "used": 167935848
      },
      "Verbose": false,
      "ObjectName": "java.lang:type=Memory"
    }
  ],
  "beans": [],
  "beans": [],
  "beans": []
}
2024-12-03T01:14:34,236 WARN  [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38585 {}] master.MasterRpcServices(626): 1a7b1a4db65e,43459,1733188427710 reported a fatal error:
***** ABORTING region server 1a7b1a4db65e,43459,1733188427710: Unrecoverable exception while closing hbase:meta,,1.1588230740 *****
Cause:
org.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.append(FSHLog.java:1191)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:1064)
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog$RingBufferEventHandler.onEvent(FSHLog.java:967)
	at com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:168)
	at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644)
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627)
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408)
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707)

2024-12-03T01:14:34,237 WARN  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.FSHLog(373): Failed sync-before-close but no outstanding appends; closing WALorg.apache.hadoop.hbase.regionserver.wal.DamagedWALException: Append sequenceId=15, requesting roll of WAL
2024-12-03T01:14:34,238 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta with entries=11, filesize=3.66 KB; new WAL /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188474222.meta
2024-12-03T01:14:34,244 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41073:41073),(127.0.0.1/127.0.0.1:45407:45407)]
2024-12-03T01:14:34,244 DEBUG [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractFSWAL(751): hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta is not closed yet, will try archiving it next time
2024-12-03T01:14:34,244 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,245 WARN  [Close-WAL-Writer-0 {}] wal.FSHLog(462): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34677,DS-01480c7a-51f6-4db0-b38b-15a0b75e8afc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-03T01:14:34,245 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta
2024-12-03T01:14:34,248 WARN  [IPC Server handler 4 on default port 40095 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta has not been closed. Lease recovery is in progress. RecoveryId = 1029 for block blk_1073741834_1016
2024-12-03T01:14:34,249 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta after 4ms
2024-12-03T01:14:34,256 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/.tmp/info/359acb413bce4a608af2ed2cb97a995f is 1080, key is row1002/info:/1733188439358/Put/seqid=0
2024-12-03T01:14:34,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741845_1030 (size=9270)
2024-12-03T01:14:34,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741845_1030 (size=9270)
2024-12-03T01:14:34,262 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/.tmp/info/359acb413bce4a608af2ed2cb97a995f
2024-12-03T01:14:34,269 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/.tmp/info/359acb413bce4a608af2ed2cb97a995f as hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/info/359acb413bce4a608af2ed2cb97a995f
2024-12-03T01:14:34,274 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/info/359acb413bce4a608af2ed2cb97a995f, entries=4, sequenceid=12, filesize=9.1 K
2024-12-03T01:14:34,275 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 836c2d63d714401836909ae12c7d605d in 46ms, sequenceid=12, compaction requested=false
2024-12-03T01:14:34,279 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/data/default/TestLogRolling-testLogRollOnPipelineRestart/836c2d63d714401836909ae12c7d605d/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1
2024-12-03T01:14:34,280 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 836c2d63d714401836909ae12c7d605d:

2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733188429346.836c2d63d714401836909ae12c7d605d.
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f733499e9a7b158639ffa297effcb71d, disabling compactions & flushes
2024-12-03T01:14:34,280 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d. after waiting 0 ms
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f733499e9a7b158639ffa297effcb71d:

2024-12-03T01:14:34,280 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,382 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:14:34,406 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-03T01:14:34,407 INFO  [regionserver/1a7b1a4db65e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-03T01:14:34,433 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3579): Received CLOSE for f733499e9a7b158639ffa297effcb71d
2024-12-03T01:14:34,433 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:14:34,433 DEBUG [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, f733499e9a7b158639ffa297effcb71d
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing f733499e9a7b158639ffa297effcb71d, disabling compactions & flushes
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:14:34,433 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,433 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d. after waiting 0 ms
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for f733499e9a7b158639ffa297effcb71d:

2024-12-03T01:14:34,433 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:14:34,434 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:meta,,1.1588230740
2024-12-03T01:14:34,434 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionServer(2803): Abort already in progress. Ignoring the current request with reason: Unrecoverable exception while closing hbase:namespace,,1733188428800.f733499e9a7b158639ffa297effcb71d.
2024-12-03T01:14:34,633 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1624): We were exiting though online regions are not empty, because some regions failed closing
2024-12-03T01:14:34,634 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,43459,1733188427710; all regions closed.
2024-12-03T01:14:34,634 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710
2024-12-03T01:14:34,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741844_1028 (size=93)
2024-12-03T01:14:34,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741844_1028 (size=93)
2024-12-03T01:14:34,986 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:35,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:36,987 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:37,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:38,238 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1016: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-03T01:14:38,249 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710/1a7b1a4db65e%2C43459%2C1733188427710.meta.1733188428746.meta after 4004ms
2024-12-03T01:14:38,250 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/WALs/1a7b1a4db65e,43459,1733188427710
2024-12-03T01:14:38,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741843_1027 (size=1162)
2024-12-03T01:14:38,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741843_1027 (size=1162)
2024-12-03T01:14:38,253 DEBUG [RS:0;1a7b1a4db65e:43459 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:14:38,253 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:14:38,253 INFO  [RS:0;1a7b1a4db65e:43459 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-03T01:14:38,253 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:14:38,254 INFO  [RS:0;1a7b1a4db65e:43459 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43459
2024-12-03T01:14:38,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,43459,1733188427710
2024-12-03T01:14:38,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:14:38,261 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,43459,1733188427710]
2024-12-03T01:14:38,261 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,43459,1733188427710; numProcessing=1
2024-12-03T01:14:38,262 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,43459,1733188427710 already deleted, retry=false
2024-12-03T01:14:38,262 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,43459,1733188427710 expired; onlineServers=0
2024-12-03T01:14:38,262 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,38585,1733188427632' *****
2024-12-03T01:14:38,262 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:14:38,263 DEBUG [M:0;1a7b1a4db65e:38585 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@619476ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:14:38,263 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,38585,1733188427632
2024-12-03T01:14:38,263 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,38585,1733188427632; all regions closed.
2024-12-03T01:14:38,263 DEBUG [M:0;1a7b1a4db65e:38585 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:14:38,263 DEBUG [M:0;1a7b1a4db65e:38585 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:14:38,263 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:14:38,263 DEBUG [M:0;1a7b1a4db65e:38585 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:14:38,263 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188428331 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188428331,5,FailOnTimeoutGroup]
2024-12-03T01:14:38,263 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188428332 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188428332,5,FailOnTimeoutGroup]
2024-12-03T01:14:38,263 INFO  [M:0;1a7b1a4db65e:38585 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:14:38,263 DEBUG [M:0;1a7b1a4db65e:38585 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:14:38,263 INFO  [M:0;1a7b1a4db65e:38585 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:14:38,263 INFO  [M:0;1a7b1a4db65e:38585 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:14:38,264 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:14:38,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:14:38,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:38,264 DEBUG [M:0;1a7b1a4db65e:38585 {}] zookeeper.ZKUtil(347): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:14:38,264 WARN  [M:0;1a7b1a4db65e:38585 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:14:38,264 INFO  [M:0;1a7b1a4db65e:38585 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:14:38,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:14:38,264 INFO  [M:0;1a7b1a4db65e:38585 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:14:38,264 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:14:38,264 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:38,264 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:38,264 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:14:38,264 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:38,265 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=40.10 KB heapSize=49.26 KB
2024-12-03T01:14:38,281 DEBUG [M:0;1a7b1a4db65e:38585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e2e32af0b74c9fa075cb33b637aad8 is 82, key is hbase:meta,,1/info:regioninfo/1733188428776/Put/seqid=0
2024-12-03T01:14:38,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741846_1031 (size=5672)
2024-12-03T01:14:38,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741846_1031 (size=5672)
2024-12-03T01:14:38,288 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e2e32af0b74c9fa075cb33b637aad8
2024-12-03T01:14:38,310 DEBUG [M:0;1a7b1a4db65e:38585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/126a1d0eb02a4ddf967ce9612c0025a4 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733188429780/Put/seqid=0
2024-12-03T01:14:38,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741847_1032 (size=7470)
2024-12-03T01:14:38,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741847_1032 (size=7470)
2024-12-03T01:14:38,320 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.50 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/126a1d0eb02a4ddf967ce9612c0025a4
2024-12-03T01:14:38,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-03T01:14:38,341 DEBUG [M:0;1a7b1a4db65e:38585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4df8e837a4894a6b93afb420fad61c02 is 69, key is 1a7b1a4db65e,43459,1733188427710/rs:state/1733188428354/Put/seqid=0
2024-12-03T01:14:38,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741848_1033 (size=5156)
2024-12-03T01:14:38,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741848_1033 (size=5156)
2024-12-03T01:14:38,347 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4df8e837a4894a6b93afb420fad61c02
2024-12-03T01:14:38,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:14:38,361 INFO  [RS:0;1a7b1a4db65e:43459 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,43459,1733188427710; zookeeper connection closed.
2024-12-03T01:14:38,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43459-0x10175e289e50001, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:14:38,361 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53f8703e {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53f8703e
2024-12-03T01:14:38,362 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T01:14:38,367 DEBUG [M:0;1a7b1a4db65e:38585 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5699c70adaaf4138b866e8784cd5901f is 52, key is load_balancer_on/state:d/1733188429339/Put/seqid=0
2024-12-03T01:14:38,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741849_1034 (size=5056)
2024-12-03T01:14:38,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741849_1034 (size=5056)
2024-12-03T01:14:38,374 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5699c70adaaf4138b866e8784cd5901f
2024-12-03T01:14:38,381 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0e2e32af0b74c9fa075cb33b637aad8 as hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0e2e32af0b74c9fa075cb33b637aad8
2024-12-03T01:14:38,386 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0e2e32af0b74c9fa075cb33b637aad8, entries=8, sequenceid=96, filesize=5.5 K
2024-12-03T01:14:38,387 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/126a1d0eb02a4ddf967ce9612c0025a4 as hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/126a1d0eb02a4ddf967ce9612c0025a4
2024-12-03T01:14:38,393 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/126a1d0eb02a4ddf967ce9612c0025a4, entries=11, sequenceid=96, filesize=7.3 K
2024-12-03T01:14:38,394 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4df8e837a4894a6b93afb420fad61c02 as hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4df8e837a4894a6b93afb420fad61c02
2024-12-03T01:14:38,398 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4df8e837a4894a6b93afb420fad61c02, entries=1, sequenceid=96, filesize=5.0 K
2024-12-03T01:14:38,399 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5699c70adaaf4138b866e8784cd5901f as hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5699c70adaaf4138b866e8784cd5901f
2024-12-03T01:14:38,404 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:40095/user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5699c70adaaf4138b866e8784cd5901f, entries=1, sequenceid=96, filesize=4.9 K
2024-12-03T01:14:38,405 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(3040): Finished flush of dataSize ~40.10 KB/41064, heapSize ~49.20 KB/50376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 140ms, sequenceid=96, compaction requested=false
2024-12-03T01:14:38,409 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:38,409 DEBUG [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:14:38,410 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/4673f485-0c02-12da-518e-d7a958050584/MasterData/WALs/1a7b1a4db65e,38585,1733188427632
2024-12-03T01:14:38,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33419 is added to blk_1073741841_1023 (size=757)
2024-12-03T01:14:38,412 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35755 is added to blk_1073741841_1023 (size=757)
2024-12-03T01:14:38,814 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:14:38,814 INFO  [M:0;1a7b1a4db65e:38585 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:14:38,814 INFO  [M:0;1a7b1a4db65e:38585 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38585
2024-12-03T01:14:38,820 DEBUG [M:0;1a7b1a4db65e:38585 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,38585,1733188427632 already deleted, retry=false
2024-12-03T01:14:38,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:14:38,922 INFO  [M:0;1a7b1a4db65e:38585 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,38585,1733188427632; zookeeper connection closed.
2024-12-03T01:14:38,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38585-0x10175e289e50000, quorum=127.0.0.1:61778, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:14:38,928 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@129adbbb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:38,929 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71144b43{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:38,929 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:38,929 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@280570d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:38,929 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1db1a42c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:38,933 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:38,933 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:38,934 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:38,934 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid 84fe90b5-7615-481c-b128-1de749075c71) service to localhost/127.0.0.1:40095
2024-12-03T01:14:38,934 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data3/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:38,935 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:38,936 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data4/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:38,941 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@532f8be9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:38,942 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1bc459ac{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:38,942 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:38,942 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52286d38{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:38,942 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d7a54ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:38,945 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:14:38,945 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:14:38,945 WARN  [BP-53554267-172.17.0.2-1733188426698 heartbeating to localhost/127.0.0.1:40095 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-53554267-172.17.0.2-1733188426698 (Datanode Uuid c191c7dc-e61b-464d-8628-7cdf6dce854a) service to localhost/127.0.0.1:40095
2024-12-03T01:14:38,945 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:14:38,946 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data1/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:38,946 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/cluster_2113df59-e96f-aa71-4b8d-874cc3e7d616/dfs/data/data2/current/BP-53554267-172.17.0.2-1733188426698 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:14:38,946 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:14:38,955 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@60016915{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:14:38,956 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@72060fdc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:14:38,956 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:14:38,957 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3082a870{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:14:38,957 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@117b2140{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir/,STOPPED}
2024-12-03T01:14:38,976 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:14:38,989 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:38,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:14:39,011 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=101 (was 87)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40095
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:40095 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:40095 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-28-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-26-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-28-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-29-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40095
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-27-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40095
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RS-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40095
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (869306163) connection to localhost/127.0.0.1:40095 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-27-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40095
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=444 (was 428) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=276 (was 399), ProcessCount=11 (was 11), AvailableMemoryMB=3714 (was 2622) - AvailableMemoryMB LEAK? -
2024-12-03T01:14:39,022 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=101, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=276, ProcessCount=11, AvailableMemoryMB=3713
2024-12-03T01:14:39,022 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:14:39,022 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.log.dir so I do NOT create it in target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c
2024-12-03T01:14:39,022 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/930f89e6-8186-1b8d-dfa3-cadd1dc583b2/hadoop.tmp.dir so I do NOT create it in target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c
2024-12-03T01:14:39,022 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870, deleteOnExit=true
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/test.cache.data in system properties and HBase conf
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:14:39,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:14:39,023 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:14:39,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:14:39,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:14:39,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:14:39,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:14:39,024 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:14:39,025 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:14:39,040 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:14:39,119 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:39,123 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:39,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:39,129 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:39,129 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:39,130 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:39,130 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e4e700f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:39,131 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f61a378{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:39,276 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@649a505c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/java.io.tmpdir/jetty-localhost-42749-hadoop-hdfs-3_4_1-tests_jar-_-any-13816012026854409842/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:14:39,277 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6ec653a7{HTTP/1.1, (http/1.1)}{localhost:42749}
2024-12-03T01:14:39,277 INFO  [Time-limited test {}] server.Server(415): Started @227276ms
2024-12-03T01:14:39,296 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:14:39,300 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,301 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,302 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,302 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,302 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,307 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,308 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,310 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,317 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,318 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,318 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,318 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,428 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:14:39,430 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,430 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,431 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,431 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,455 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,456 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,456 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,457 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,457 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,458 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,463 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,463 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,464 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,467 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:39,480 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:39,485 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:39,489 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:39,489 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:39,489 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:39,490 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@903456d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:39,490 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75e97d20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:39,663 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4fe74ba6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/java.io.tmpdir/jetty-localhost-42705-hadoop-hdfs-3_4_1-tests_jar-_-any-11508110143306646225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:39,664 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@402c6ca9{HTTP/1.1, (http/1.1)}{localhost:42705}
2024-12-03T01:14:39,664 INFO  [Time-limited test {}] server.Server(415): Started @227663ms
2024-12-03T01:14:39,666 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:39,743 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:14:39,747 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:14:39,754 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:14:39,755 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:14:39,755 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:14:39,755 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59bc5232{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:14:39,756 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c7be892{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:14:39,839 WARN  [Thread-1371 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data1/current/BP-345640683-172.17.0.2-1733188479059/current, will proceed with Du for space computation calculation, 
2024-12-03T01:14:39,840 WARN  [Thread-1372 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data2/current/BP-345640683-172.17.0.2-1733188479059/current, will proceed with Du for space computation calculation, 
2024-12-03T01:14:39,877 WARN  [Thread-1350 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:39,881 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x286fd201cd17e707 with lease ID 0xc8c7ced593144f55: Processing first storage report for DS-3c2206b5-281c-45ca-bff4-7da2a27d7d80 from datanode DatanodeRegistration(127.0.0.1:44957, datanodeUuid=4f940019-7dd2-419c-8f04-7e3802432550, infoPort=38019, infoSecurePort=0, ipcPort=39079, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059)
2024-12-03T01:14:39,881 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x286fd201cd17e707 with lease ID 0xc8c7ced593144f55: from storage DS-3c2206b5-281c-45ca-bff4-7da2a27d7d80 node DatanodeRegistration(127.0.0.1:44957, datanodeUuid=4f940019-7dd2-419c-8f04-7e3802432550, infoPort=38019, infoSecurePort=0, ipcPort=39079, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:39,881 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x286fd201cd17e707 with lease ID 0xc8c7ced593144f55: Processing first storage report for DS-79d4bc33-176a-442d-91ca-9bd9f7c5b67c from datanode DatanodeRegistration(127.0.0.1:44957, datanodeUuid=4f940019-7dd2-419c-8f04-7e3802432550, infoPort=38019, infoSecurePort=0, ipcPort=39079, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059)
2024-12-03T01:14:39,881 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x286fd201cd17e707 with lease ID 0xc8c7ced593144f55: from storage DS-79d4bc33-176a-442d-91ca-9bd9f7c5b67c node DatanodeRegistration(127.0.0.1:44957, datanodeUuid=4f940019-7dd2-419c-8f04-7e3802432550, infoPort=38019, infoSecurePort=0, ipcPort=39079, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:39,953 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea2a712{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/java.io.tmpdir/jetty-localhost-33251-hadoop-hdfs-3_4_1-tests_jar-_-any-3460466027998653555/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:14:39,954 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fa2966d{HTTP/1.1, (http/1.1)}{localhost:33251}
2024-12-03T01:14:39,954 INFO  [Time-limited test {}] server.Server(415): Started @227953ms
2024-12-03T01:14:39,955 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:14:39,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:40,120 WARN  [Thread-1398 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data4/current/BP-345640683-172.17.0.2-1733188479059/current, will proceed with Du for space computation calculation, 
2024-12-03T01:14:40,121 WARN  [Thread-1397 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data3/current/BP-345640683-172.17.0.2-1733188479059/current, will proceed with Du for space computation calculation, 
2024-12-03T01:14:40,161 WARN  [Thread-1386 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:14:40,164 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa08e800c04e56788 with lease ID 0xc8c7ced593144f56: Processing first storage report for DS-42554357-38c9-4c1d-a1e6-5dbd484fc97c from datanode DatanodeRegistration(127.0.0.1:33189, datanodeUuid=6fa99f82-1612-4180-9130-7d238f5a8c2d, infoPort=33621, infoSecurePort=0, ipcPort=37739, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059)
2024-12-03T01:14:40,164 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa08e800c04e56788 with lease ID 0xc8c7ced593144f56: from storage DS-42554357-38c9-4c1d-a1e6-5dbd484fc97c node DatanodeRegistration(127.0.0.1:33189, datanodeUuid=6fa99f82-1612-4180-9130-7d238f5a8c2d, infoPort=33621, infoSecurePort=0, ipcPort=37739, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:14:40,164 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa08e800c04e56788 with lease ID 0xc8c7ced593144f56: Processing first storage report for DS-1188d3e9-6d4c-4d7a-b62f-d2d3226fc1cf from datanode DatanodeRegistration(127.0.0.1:33189, datanodeUuid=6fa99f82-1612-4180-9130-7d238f5a8c2d, infoPort=33621, infoSecurePort=0, ipcPort=37739, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059)
2024-12-03T01:14:40,164 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa08e800c04e56788 with lease ID 0xc8c7ced593144f56: from storage DS-1188d3e9-6d4c-4d7a-b62f-d2d3226fc1cf node DatanodeRegistration(127.0.0.1:33189, datanodeUuid=6fa99f82-1612-4180-9130-7d238f5a8c2d, infoPort=33621, infoSecurePort=0, ipcPort=37739, storageInfo=lv=-57;cid=testClusterID;nsid=1793993286;c=1733188479059), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:14:40,211 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c
2024-12-03T01:14:40,214 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/zookeeper_0, clientPort=63635, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:14:40,215 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=63635
2024-12-03T01:14:40,216 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,218 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:14:40,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:14:40,637 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9 with version=8
2024-12-03T01:14:40,637 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:14:40,639 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:14:40,640 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:14:40,640 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:42773
2024-12-03T01:14:40,641 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,642 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,645 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:42773 connecting to ZooKeeper ensemble=127.0.0.1:63635
2024-12-03T01:14:40,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427730x0, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:14:40,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42773-0x10175e357650000 connected
2024-12-03T01:14:40,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:14:40,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:14:40,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:14:40,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42773
2024-12-03T01:14:40,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42773
2024-12-03T01:14:40,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42773
2024-12-03T01:14:40,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42773
2024-12-03T01:14:40,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42773
2024-12-03T01:14:40,688 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9, hbase.cluster.distributed=false
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:14:40,705 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:14:40,706 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44179
2024-12-03T01:14:40,706 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:14:40,707 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:14:40,707 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,709 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,712 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:44179 connecting to ZooKeeper ensemble=127.0.0.1:63635
2024-12-03T01:14:40,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:441790x0, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:14:40,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44179-0x10175e357650001 connected
2024-12-03T01:14:40,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:14:40,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:14:40,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:14:40,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44179
2024-12-03T01:14:40,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44179
2024-12-03T01:14:40,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44179
2024-12-03T01:14:40,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44179
2024-12-03T01:14:40,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44179
2024-12-03T01:14:40,718 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:14:40,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:14:40,721 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:14:40,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:14:40,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,723 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:14:40,723 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,42773,1733188480639 from backup master directory
2024-12-03T01:14:40,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:14:40,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:14:40,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:14:40,727 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:14:40,727 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,734 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:42773
2024-12-03T01:14:40,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:14:40,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:14:40,742 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/hbase.id with ID: dae09776-268c-48db-88c2-29d56e99b40f
2024-12-03T01:14:40,752 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:40,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:14:40,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:14:40,767 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:14:40,767 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:14:40,768 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:14:40,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:14:40,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:14:40,782 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store
2024-12-03T01:14:40,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:14:40,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:14:40,795 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:40,796 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:14:40,796 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:40,796 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:40,796 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:14:40,796 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:40,796 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:14:40,796 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:14:40,797 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/.initializing
2024-12-03T01:14:40,797 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/WALs/1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,800 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C42773%2C1733188480639, suffix=, logDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/WALs/1a7b1a4db65e,42773,1733188480639, archiveDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/oldWALs, maxLogs=10
2024-12-03T01:14:40,801 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C42773%2C1733188480639.1733188480801
2024-12-03T01:14:40,815 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/WALs/1a7b1a4db65e,42773,1733188480639/1a7b1a4db65e%2C42773%2C1733188480639.1733188480801
2024-12-03T01:14:40,815 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33621:33621),(127.0.0.1/127.0.0.1:38019:38019)]
2024-12-03T01:14:40,815 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:14:40,815 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:40,815 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:14:40,815 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:14:40,819 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:14:40,821 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:14:40,822 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:40,822 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:40,822 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:14:40,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:14:40,824 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:40,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:14:40,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:14:40,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:14:40,826 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:40,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:14:40,827 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:14:40,828 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:14:40,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:40,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:14:40,830 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:14:40,830 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:14:40,832 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:14:40,833 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:14:40,836 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:14:40,837 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=750133, jitterRate=-0.046157076954841614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:14:40,838 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:14:40,840 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:14:40,844 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25af6777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:14:40,845 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:14:40,846 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:14:40,846 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:14:40,846 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:14:40,847 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T01:14:40,847 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-03T01:14:40,847 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:14:40,853 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:14:40,854 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:14:40,855 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:14:40,855 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:14:40,856 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:14:40,857 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:14:40,857 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:14:40,859 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:14:40,862 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:14:40,862 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:14:40,864 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:14:40,865 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:14:40,866 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:14:40,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:14:40,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:14:40,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,870 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,42773,1733188480639, sessionid=0x10175e357650000, setting cluster-up flag (Was=false)
2024-12-03T01:14:40,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,880 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:14:40,881 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:40,891 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:14:40,892 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:40,894 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:14:40,894 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:14:40,894 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,42773,1733188480639
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:14:40,895 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188510896
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:14:40,896 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:14:40,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,897 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:14:40,897 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:14:40,898 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188480897,5,FailOnTimeoutGroup]
2024-12-03T01:14:40,898 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188480898,5,FailOnTimeoutGroup]
2024-12-03T01:14:40,898 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,898 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:14:40,898 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,898 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,898 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:40,898 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:14:40,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:14:40,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:14:40,930 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:44179
2024-12-03T01:14:40,931 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1008): ClusterId : dae09776-268c-48db-88c2-29d56e99b40f
2024-12-03T01:14:40,931 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:14:40,934 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:14:40,934 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:14:40,936 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:14:40,936 DEBUG [RS:0;1a7b1a4db65e:44179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56475a74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:14:40,936 DEBUG [RS:0;1a7b1a4db65e:44179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540b25d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:14:40,936 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:14:40,936 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:14:40,936 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:14:40,937 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,42773,1733188480639 with isa=1a7b1a4db65e/172.17.0.2:44179, startcode=1733188480704
2024-12-03T01:14:40,937 DEBUG [RS:0;1a7b1a4db65e:44179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:14:40,939 INFO  [RS-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44659, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:14:40,940 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42773 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,940 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42773 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,941 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9
2024-12-03T01:14:40,941 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:33065
2024-12-03T01:14:40,941 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:14:40,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:14:40,945 DEBUG [RS:0;1a7b1a4db65e:44179 {}] zookeeper.ZKUtil(111): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,945 WARN  [RS:0;1a7b1a4db65e:44179 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:14:40,945 INFO  [RS:0;1a7b1a4db65e:44179 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:14:40,945 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,945 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,44179,1733188480704]
2024-12-03T01:14:40,949 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:14:40,949 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:14:40,952 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:14:40,953 INFO  [RS:0;1a7b1a4db65e:44179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:14:40,953 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,953 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:14:40,954 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:14:40,954 DEBUG [RS:0;1a7b1a4db65e:44179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:14:40,956 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,957 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,957 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,957 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,957 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44179,1733188480704-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:14:40,974 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:14:40,974 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44179,1733188480704-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:40,989 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.Replication(204): 1a7b1a4db65e,44179,1733188480704 started
2024-12-03T01:14:40,989 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,44179,1733188480704, RpcServer on 1a7b1a4db65e/172.17.0.2:44179, sessionid=0x10175e357650001
2024-12-03T01:14:40,989 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:14:40,989 DEBUG [RS:0;1a7b1a4db65e:44179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,989 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,44179,1733188480704'
2024-12-03T01:14:40,989 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:14:40,989 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,44179,1733188480704'
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:14:40,990 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:14:40,991 DEBUG [RS:0;1a7b1a4db65e:44179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:14:40,991 INFO  [RS:0;1a7b1a4db65e:44179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:14:40,991 INFO  [RS:0;1a7b1a4db65e:44179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:14:40,993 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:41,093 INFO  [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C44179%2C1733188480704, suffix=, logDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704, archiveDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs, maxLogs=32
2024-12-03T01:14:41,094 INFO  [RS:0;1a7b1a4db65e:44179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44179%2C1733188480704.1733188481094
2024-12-03T01:14:41,105 INFO  [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188481094
2024-12-03T01:14:41,106 DEBUG [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:33621:33621)]
2024-12-03T01:14:41,312 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:14:41,312 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9
2024-12-03T01:14:41,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741833_1009 (size=32)
2024-12-03T01:14:41,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741833_1009 (size=32)
2024-12-03T01:14:41,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:41,330 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:14:41,332 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:14:41,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,333 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,333 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:14:41,335 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:14:41,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:14:41,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:14:41,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,342 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740
2024-12-03T01:14:41,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740
2024-12-03T01:14:41,345 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:14:41,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:14:41,349 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:14:41,350 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=808683, jitterRate=0.028293833136558533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:14:41,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:14:41,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:14:41,350 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:14:41,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:14:41,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:14:41,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:14:41,351 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:14:41,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:14:41,352 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:14:41,352 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:14:41,352 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:14:41,354 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:14:41,355 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:14:41,505 DEBUG [1a7b1a4db65e:42773 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:14:41,505 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:41,507 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,44179,1733188480704, state=OPENING
2024-12-03T01:14:41,508 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:14:41,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:41,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:41,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:14:41,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:14:41,511 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,44179,1733188480704}]
2024-12-03T01:14:41,664 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:41,664 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:14:41,667 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32826, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:14:41,672 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:14:41,672 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:14:41,674 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C44179%2C1733188480704.meta, suffix=.meta, logDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704, archiveDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs, maxLogs=32
2024-12-03T01:14:41,675 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44179%2C1733188480704.meta.1733188481675.meta
2024-12-03T01:14:41,686 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.meta.1733188481675.meta
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:33621:33621)]
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:14:41,686 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:14:41,686 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:41,687 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:14:41,687 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:14:41,688 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:14:41,689 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:14:41,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,689 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,690 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:14:41,690 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:14:41,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,691 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,691 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:14:41,692 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:14:41,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,692 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:14:41,693 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740
2024-12-03T01:14:41,694 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740
2024-12-03T01:14:41,696 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:14:41,697 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:14:41,698 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805080, jitterRate=0.02371305227279663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:14:41,698 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:14:41,699 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188481664
2024-12-03T01:14:41,700 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:14:41,701 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:14:41,701 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:41,702 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,44179,1733188480704, state=OPEN
2024-12-03T01:14:41,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:14:41,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:14:41,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:14:41,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:14:41,712 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:14:41,712 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,44179,1733188480704 in 199 msec
2024-12-03T01:14:41,714 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:14:41,714 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 360 msec
2024-12-03T01:14:41,716 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 822 msec
2024-12-03T01:14:41,716 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188481716, completionTime=-1
2024-12-03T01:14:41,716 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:14:41,716 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:14:41,717 DEBUG [hconnection-0x3e8c0b0f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:14:41,719 INFO  [RS-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32836, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:14:41,720 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:14:41,720 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188541720
2024-12-03T01:14:41,720 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188601720
2024-12-03T01:14:41,720 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:42773, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:14:41,726 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:14:41,728 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:14:41,728 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:14:41,731 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:14:41,731 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:41,732 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:14:41,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:14:41,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:14:41,741 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0d06b76a38c879ff50377898eba23cc6, NAME => 'hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9
2024-12-03T01:14:41,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:14:41,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 0d06b76a38c879ff50377898eba23cc6, disabling compactions & flushes
2024-12-03T01:14:41,754 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6. after waiting 0 ms
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:41,754 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:41,754 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0d06b76a38c879ff50377898eba23cc6:

2024-12-03T01:14:41,756 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:14:41,756 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188481756"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188481756"}]},"ts":"1733188481756"}
2024-12-03T01:14:41,758 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:14:41,759 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:14:41,759 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188481759"}]},"ts":"1733188481759"}
2024-12-03T01:14:41,761 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:14:41,764 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0d06b76a38c879ff50377898eba23cc6, ASSIGN}]
2024-12-03T01:14:41,765 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0d06b76a38c879ff50377898eba23cc6, ASSIGN
2024-12-03T01:14:41,766 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=0d06b76a38c879ff50377898eba23cc6, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,44179,1733188480704; forceNewPlan=false, retain=false
2024-12-03T01:14:41,917 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0d06b76a38c879ff50377898eba23cc6, regionState=OPENING, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:41,919 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 0d06b76a38c879ff50377898eba23cc6, server=1a7b1a4db65e,44179,1733188480704}]
2024-12-03T01:14:41,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:42,072 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:42,076 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:42,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 0d06b76a38c879ff50377898eba23cc6, NAME => 'hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:14:42,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:42,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,077 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,078 INFO  [StoreOpener-0d06b76a38c879ff50377898eba23cc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0d06b76a38c879ff50377898eba23cc6 
2024-12-03T01:14:42,079 INFO  [StoreOpener-0d06b76a38c879ff50377898eba23cc6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d06b76a38c879ff50377898eba23cc6 columnFamilyName info
2024-12-03T01:14:42,079 DEBUG [StoreOpener-0d06b76a38c879ff50377898eba23cc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:42,080 INFO  [StoreOpener-0d06b76a38c879ff50377898eba23cc6-1 {}] regionserver.HStore(327): Store=0d06b76a38c879ff50377898eba23cc6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:14:42,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:14:42,085 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:14:42,086 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 0d06b76a38c879ff50377898eba23cc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=801773, jitterRate=0.019508257508277893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:14:42,086 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 0d06b76a38c879ff50377898eba23cc6:

2024-12-03T01:14:42,086 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6., pid=6, masterSystemTime=1733188482071
2024-12-03T01:14:42,088 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:42,088 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:42,089 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0d06b76a38c879ff50377898eba23cc6, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:42,093 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:14:42,093 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 0d06b76a38c879ff50377898eba23cc6, server=1a7b1a4db65e,44179,1733188480704 in 172 msec
2024-12-03T01:14:42,095 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:14:42,095 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=0d06b76a38c879ff50377898eba23cc6, ASSIGN in 329 msec
2024-12-03T01:14:42,096 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:14:42,096 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188482096"}]},"ts":"1733188482096"}
2024-12-03T01:14:42,098 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:14:42,101 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:14:42,102 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 374 msec
2024-12-03T01:14:42,129 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:14:42,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:42,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:14:42,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:14:42,138 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:14:42,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:14:42,157 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 19 msec
2024-12-03T01:14:42,160 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:14:42,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:14:42,173 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 13 msec
2024-12-03T01:14:42,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:14:42,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:14:42,189 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.462sec
2024-12-03T01:14:42,189 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:14:42,190 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:14:42,192 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:14:42,192 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:14:42,192 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,42773,1733188480639-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:14:42,221 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b6c4a6a to 127.0.0.1:63635 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51ffb791
2024-12-03T01:14:42,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59630237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:14:42,229 DEBUG [hconnection-0x6887e319-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:14:42,230 INFO  [RS-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32850, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:14:42,232 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,42773,1733188480639
2024-12-03T01:14:42,232 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:14:42,236 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:14:42,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T01:14:42,239 INFO  [RS-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49254, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T01:14:42,240 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T01:14:42,240 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T01:14:42,241 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:14:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:14:42,243 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:14:42,244 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:42,244 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 9
2024-12-03T01:14:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:14:42,245 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:14:42,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741837_1013 (size=405)
2024-12-03T01:14:42,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741837_1013 (size=405)
2024-12-03T01:14:42,662 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => c01e28d9af22e6a31ddf3338064570cf, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9
2024-12-03T01:14:42,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741838_1014 (size=88)
2024-12-03T01:14:42,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741838_1014 (size=88)
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1681): Closing c01e28d9af22e6a31ddf3338064570cf, disabling compactions & flushes
2024-12-03T01:14:42,671 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf. after waiting 0 ms
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:42,671 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:42,671 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:14:42,673 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:14:42,673 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733188482673"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188482673"}]},"ts":"1733188482673"}
2024-12-03T01:14:42,675 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:14:42,676 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:14:42,676 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188482676"}]},"ts":"1733188482676"}
2024-12-03T01:14:42,677 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-03T01:14:42,682 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=c01e28d9af22e6a31ddf3338064570cf, ASSIGN}]
2024-12-03T01:14:42,683 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=c01e28d9af22e6a31ddf3338064570cf, ASSIGN
2024-12-03T01:14:42,684 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=c01e28d9af22e6a31ddf3338064570cf, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,44179,1733188480704; forceNewPlan=false, retain=false
2024-12-03T01:14:42,835 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c01e28d9af22e6a31ddf3338064570cf, regionState=OPENING, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:42,837 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure c01e28d9af22e6a31ddf3338064570cf, server=1a7b1a4db65e,44179,1733188480704}]
2024-12-03T01:14:42,989 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:42,995 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:42,995 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => c01e28d9af22e6a31ddf3338064570cf, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:14:42,995 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:42,996 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:42,996 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:14:42,996 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:42,996 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:43,001 INFO  [StoreOpener-c01e28d9af22e6a31ddf3338064570cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c01e28d9af22e6a31ddf3338064570cf 
2024-12-03T01:14:43,003 INFO  [StoreOpener-c01e28d9af22e6a31ddf3338064570cf-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c01e28d9af22e6a31ddf3338064570cf columnFamilyName info
2024-12-03T01:14:43,003 DEBUG [StoreOpener-c01e28d9af22e6a31ddf3338064570cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:14:43,004 INFO  [StoreOpener-c01e28d9af22e6a31ddf3338064570cf-1 {}] regionserver.HStore(327): Store=c01e28d9af22e6a31ddf3338064570cf/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:14:43,005 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:43,005 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:43,007 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:14:43,010 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:14:43,010 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened c01e28d9af22e6a31ddf3338064570cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785117, jitterRate=-0.0016723275184631348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:14:43,011 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:14:43,016 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf., pid=11, masterSystemTime=1733188482989
2024-12-03T01:14:43,020 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:43,020 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:43,020 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=c01e28d9af22e6a31ddf3338064570cf, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:43,025 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-03T01:14:43,025 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure c01e28d9af22e6a31ddf3338064570cf, server=1a7b1a4db65e,44179,1733188480704 in 186 msec
2024-12-03T01:14:43,027 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T01:14:43,027 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=c01e28d9af22e6a31ddf3338064570cf, ASSIGN in 343 msec
2024-12-03T01:14:43,028 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:14:43,028 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188483028"}]},"ts":"1733188483028"}
2024-12-03T01:14:43,029 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-03T01:14:43,034 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:14:43,036 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 794 msec
2024-12-03T01:14:43,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:44,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:45,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:46,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:47,188 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:14:47,189 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,189 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,190 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,204 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,204 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,204 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,204 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,205 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,210 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:14:47,218 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-03T01:14:47,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:48,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:14:48,331 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T01:14:48,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:49,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta after 68054ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:14:50,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:51,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:52,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:14:52,247 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 9 completed
2024-12-03T01:14:52,249 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:14:52,249 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:14:52,258 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush hbase:namespace
2024-12-03T01:14:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace
2024-12-03T01:14:52,265 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_PREPARE
2024-12-03T01:14:52,266 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T01:14:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-03T01:14:52,268 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T01:14:52,427 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:14:52,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44179 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13
2024-12-03T01:14:52,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:52,430 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing 0d06b76a38c879ff50377898eba23cc6 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:14:52,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/.tmp/info/994c3ebc8f0a47379399380c961df3da is 45, key is default/info:d/1733188482146/Put/seqid=0
2024-12-03T01:14:52,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741839_1015 (size=5037)
2024-12-03T01:14:52,459 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741839_1015 (size=5037)
2024-12-03T01:14:52,461 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/.tmp/info/994c3ebc8f0a47379399380c961df3da
2024-12-03T01:14:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/.tmp/info/994c3ebc8f0a47379399380c961df3da as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/info/994c3ebc8f0a47379399380c961df3da
2024-12-03T01:14:52,474 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/info/994c3ebc8f0a47379399380c961df3da, entries=2, sequenceid=6, filesize=4.9 K
2024-12-03T01:14:52,475 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 0d06b76a38c879ff50377898eba23cc6 in 45ms, sequenceid=6, compaction requested=false
2024-12-03T01:14:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for 0d06b76a38c879ff50377898eba23cc6:

2024-12-03T01:14:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:14:52,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13
2024-12-03T01:14:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster(4106): Remote procedure done, pid=13
2024-12-03T01:14:52,482 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-03T01:14:52,482 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 212 msec
2024-12-03T01:14:52,483 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=hbase:namespace in 222 msec
2024-12-03T01:14:53,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:54,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:55,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:56,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:57,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:58,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:14:59,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:00,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:01,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:02,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12
2024-12-03T01:15:02,268 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: hbase:namespace, procId: 12 completed
2024-12-03T01:15:02,278 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-03T01:15:02,280 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T01:15:02,281 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T01:15:02,281 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T01:15:02,433 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:02,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44179 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15
2024-12-03T01:15:02,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:02,435 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing c01e28d9af22e6a31ddf3338064570cf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T01:15:02,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ef1096c3a0b445eb91d28d355663371d is 1080, key is row0001/info:/1733188502272/Put/seqid=0
2024-12-03T01:15:02,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741840_1016 (size=6033)
2024-12-03T01:15:02,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741840_1016 (size=6033)
2024-12-03T01:15:02,472 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ef1096c3a0b445eb91d28d355663371d
2024-12-03T01:15:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ef1096c3a0b445eb91d28d355663371d as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d
2024-12-03T01:15:02,490 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d, entries=1, sequenceid=5, filesize=5.9 K
2024-12-03T01:15:02,491 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c01e28d9af22e6a31ddf3338064570cf in 56ms, sequenceid=5, compaction requested=false
2024-12-03T01:15:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15
2024-12-03T01:15:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster(4106): Remote procedure done, pid=15
2024-12-03T01:15:02,495 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14
2024-12-03T01:15:02,495 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 212 msec
2024-12-03T01:15:02,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 218 msec
2024-12-03T01:15:03,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:04,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:05,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:06,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:07,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:08,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:09,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:09,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 23930
2024-12-03T01:15:10,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:10,210 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:15:11,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:12,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14
2024-12-03T01:15:12,281 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 14 completed
2024-12-03T01:15:12,287 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:12,289 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T01:15:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-03T01:15:12,290 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T01:15:12,290 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T01:15:12,442 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:12,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44179 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17
2024-12-03T01:15:12,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:12,444 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing c01e28d9af22e6a31ddf3338064570cf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T01:15:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/41de361a84f44cdc9cdb88b4166c1c49 is 1080, key is row0002/info:/1733188512282/Put/seqid=0
2024-12-03T01:15:12,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741841_1017 (size=6033)
2024-12-03T01:15:12,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741841_1017 (size=6033)
2024-12-03T01:15:12,472 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/41de361a84f44cdc9cdb88b4166c1c49
2024-12-03T01:15:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/41de361a84f44cdc9cdb88b4166c1c49 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49
2024-12-03T01:15:12,502 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49, entries=1, sequenceid=9, filesize=5.9 K
2024-12-03T01:15:12,509 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c01e28d9af22e6a31ddf3338064570cf in 66ms, sequenceid=9, compaction requested=false
2024-12-03T01:15:12,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17
2024-12-03T01:15:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster(4106): Remote procedure done, pid=17
2024-12-03T01:15:12,585 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16
2024-12-03T01:15:12,585 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 286 msec
2024-12-03T01:15:12,592 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 300 msec
2024-12-03T01:15:12,936 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:15:12,938 INFO  [RS-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53914, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:15:13,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:14,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:15,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:16,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:17,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:18,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:19,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:20,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:21,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:22,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:22,214 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T01:15:22,214 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T01:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16
2024-12-03T01:15:22,290 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 16 completed
2024-12-03T01:15:22,293 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44179%2C1733188480704.1733188522293
2024-12-03T01:15:22,302 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188481094 with entries=13, filesize=6.41 KB; new WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188522293
2024-12-03T01:15:22,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:33621:33621)]
2024-12-03T01:15:22,302 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188481094 is not closed yet, will try archiving it next time
2024-12-03T01:15:22,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741832_1008 (size=6574)
2024-12-03T01:15:22,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741832_1008 (size=6574)
2024-12-03T01:15:22,306 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-03T01:15:22,310 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T01:15:22,311 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T01:15:22,311 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T01:15:22,463 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:22,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44179 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19
2024-12-03T01:15:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:22,465 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing c01e28d9af22e6a31ddf3338064570cf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T01:15:22,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/8e6e5fc41713440ca9041c66925a1cd0 is 1080, key is row0003/info:/1733188522291/Put/seqid=0
2024-12-03T01:15:22,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741843_1019 (size=6033)
2024-12-03T01:15:22,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741843_1019 (size=6033)
2024-12-03T01:15:22,883 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/8e6e5fc41713440ca9041c66925a1cd0
2024-12-03T01:15:22,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/8e6e5fc41713440ca9041c66925a1cd0 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0
2024-12-03T01:15:22,897 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0, entries=1, sequenceid=13, filesize=5.9 K
2024-12-03T01:15:22,898 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c01e28d9af22e6a31ddf3338064570cf in 434ms, sequenceid=13, compaction requested=true
2024-12-03T01:15:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:22,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19
2024-12-03T01:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster(4106): Remote procedure done, pid=19
2024-12-03T01:15:22,901 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18
2024-12-03T01:15:22,901 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 589 msec
2024-12-03T01:15:22,903 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 596 msec
2024-12-03T01:15:23,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:24,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:25,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:26,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:27,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:27,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 0d06b76a38c879ff50377898eba23cc6, had cached 0 bytes from a total of 5037
2024-12-03T01:15:27,996 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region c01e28d9af22e6a31ddf3338064570cf, had cached 0 bytes from a total of 18099
2024-12-03T01:15:28,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:29,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:30,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:31,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:32,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18
2024-12-03T01:15:32,311 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 18 completed
2024-12-03T01:15:32,311 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:15:32,312 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:15:32,313 DEBUG [Time-limited test {}] regionserver.HStore(1540): c01e28d9af22e6a31ddf3338064570cf/info is initiating minor compaction (all files)
2024-12-03T01:15:32,313 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:15:32,313 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:32,313 INFO  [Time-limited test {}] regionserver.HRegion(2351): Starting compaction of c01e28d9af22e6a31ddf3338064570cf/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:32,313 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0] into tmpdir=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp, totalSize=17.7 K
2024-12-03T01:15:32,313 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting ef1096c3a0b445eb91d28d355663371d, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733188502272
2024-12-03T01:15:32,314 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 41de361a84f44cdc9cdb88b4166c1c49, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733188512282
2024-12-03T01:15:32,314 DEBUG [Time-limited test {}] compactions.Compactor(224): Compacting 8e6e5fc41713440ca9041c66925a1cd0, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733188522291
2024-12-03T01:15:32,325 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): c01e28d9af22e6a31ddf3338064570cf#info#compaction#29 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:15:32,326 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/e9ef139fe87f4cd1bf102af516c8a74e is 1080, key is row0001/info:/1733188502272/Put/seqid=0
2024-12-03T01:15:32,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741844_1020 (size=8296)
2024-12-03T01:15:32,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741844_1020 (size=8296)
2024-12-03T01:15:32,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/e9ef139fe87f4cd1bf102af516c8a74e as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/e9ef139fe87f4cd1bf102af516c8a74e
2024-12-03T01:15:32,344 INFO  [Time-limited test {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in c01e28d9af22e6a31ddf3338064570cf/info of c01e28d9af22e6a31ddf3338064570cf into e9ef139fe87f4cd1bf102af516c8a74e(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:15:32,344 DEBUG [Time-limited test {}] regionserver.HRegion(2381): Compaction status journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:32,346 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44179%2C1733188480704.1733188532346
2024-12-03T01:15:32,354 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188522293 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188532346
2024-12-03T01:15:32,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33621:33621),(127.0.0.1/127.0.0.1:38019:38019)]
2024-12-03T01:15:32,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188522293 is not closed yet, will try archiving it next time
2024-12-03T01:15:32,354 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188481094 to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs/1a7b1a4db65e%2C44179%2C1733188480704.1733188481094
2024-12-03T01:15:32,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741842_1018 (size=2520)
2024-12-03T01:15:32,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741842_1018 (size=2520)
2024-12-03T01:15:32,358 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-03T01:15:32,360 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-03T01:15:32,361 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-03T01:15:32,361 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-03T01:15:32,512 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:32,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44179 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21
2024-12-03T01:15:32,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:32,514 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing c01e28d9af22e6a31ddf3338064570cf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T01:15:32,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/6f0a48ef798840e497976a7694788d7f is 1080, key is row0000/info:/1733188532345/Put/seqid=0
2024-12-03T01:15:32,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741846_1022 (size=6033)
2024-12-03T01:15:32,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741846_1022 (size=6033)
2024-12-03T01:15:32,525 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/6f0a48ef798840e497976a7694788d7f
2024-12-03T01:15:32,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/6f0a48ef798840e497976a7694788d7f as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/6f0a48ef798840e497976a7694788d7f
2024-12-03T01:15:32,536 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/6f0a48ef798840e497976a7694788d7f, entries=1, sequenceid=18, filesize=5.9 K
2024-12-03T01:15:32,537 INFO  [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c01e28d9af22e6a31ddf3338064570cf in 24ms, sequenceid=18, compaction requested=false
2024-12-03T01:15:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:32,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21
2024-12-03T01:15:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.HMaster(4106): Remote procedure done, pid=21
2024-12-03T01:15:32,541 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20
2024-12-03T01:15:32,541 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec
2024-12-03T01:15:32,543 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-12-03T01:15:33,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:34,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:35,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:36,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:37,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:38,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:39,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:40,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:40,211 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:15:41,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:42,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:42,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42773 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20
2024-12-03T01:15:42,361 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling, procId: 20 completed
2024-12-03T01:15:42,364 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44179%2C1733188480704.1733188542363
2024-12-03T01:15:42,370 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188532346 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188542363
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:33621:33621)]
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188532346 is not closed yet, will try archiving it next time
2024-12-03T01:15:42,371 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:15:42,371 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704/1a7b1a4db65e%2C44179%2C1733188480704.1733188522293 to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs/1a7b1a4db65e%2C44179%2C1733188480704.1733188522293
2024-12-03T01:15:42,371 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b6c4a6a to 127.0.0.1:63635
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:15:42,371 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=296570535, stopped=false
2024-12-03T01:15:42,371 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,42773,1733188480639
2024-12-03T01:15:42,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741845_1021 (size=2026)
2024-12-03T01:15:42,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741845_1021 (size=2026)
2024-12-03T01:15:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:15:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:15:42,373 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:15:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:42,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:42,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:15:42,373 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,44179,1733188480704' *****
2024-12-03T01:15:42,373 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:15:42,374 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:15:42,374 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:15:42,374 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:15:42,374 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:15:42,374 INFO  [RS:0;1a7b1a4db65e:44179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(3579): Received CLOSE for c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(3579): Received CLOSE for 0d06b76a38c879ff50377898eba23cc6
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:42,375 DEBUG [RS:0;1a7b1a4db65e:44179 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:15:42,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c01e28d9af22e6a31ddf3338064570cf, disabling compactions & flushes
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:15:42,375 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:15:42,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:42,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf. after waiting 0 ms
2024-12-03T01:15:42,375 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:15:42,375 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing c01e28d9af22e6a31ddf3338064570cf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-03T01:15:42,375 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1599): Waiting on 3 regions to close
2024-12-03T01:15:42,375 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1603): Online Regions={c01e28d9af22e6a31ddf3338064570cf=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf., 1588230740=hbase:meta,,1.1588230740, 0d06b76a38c879ff50377898eba23cc6=hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.}
2024-12-03T01:15:42,375 DEBUG [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1629): Waiting on 0d06b76a38c879ff50377898eba23cc6, 1588230740, c01e28d9af22e6a31ddf3338064570cf
2024-12-03T01:15:42,375 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:15:42,376 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:15:42,376 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:15:42,376 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:15:42,376 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:15:42,376 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=3.05 KB heapSize=5.55 KB
2024-12-03T01:15:42,379 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ed3a134894e1405e9681598ea3f93f40 is 1080, key is row0001/info:/1733188542362/Put/seqid=0
2024-12-03T01:15:42,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741848_1024 (size=6033)
2024-12-03T01:15:42,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741848_1024 (size=6033)
2024-12-03T01:15:42,391 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ed3a134894e1405e9681598ea3f93f40
2024-12-03T01:15:42,393 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/info/e04d5bbdcf074c93b98a3ab035da9844 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf./info:regioninfo/1733188483020/Put/seqid=0
2024-12-03T01:15:42,397 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/.tmp/info/ed3a134894e1405e9681598ea3f93f40 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ed3a134894e1405e9681598ea3f93f40
2024-12-03T01:15:42,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741849_1025 (size=8430)
2024-12-03T01:15:42,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741849_1025 (size=8430)
2024-12-03T01:15:42,403 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.79 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/info/e04d5bbdcf074c93b98a3ab035da9844
2024-12-03T01:15:42,403 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ed3a134894e1405e9681598ea3f93f40, entries=1, sequenceid=22, filesize=5.9 K
2024-12-03T01:15:42,404 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for c01e28d9af22e6a31ddf3338064570cf in 29ms, sequenceid=22, compaction requested=true
2024-12-03T01:15:42,405 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0] to archive
2024-12-03T01:15:42,406 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:15:42,407 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/ef1096c3a0b445eb91d28d355663371d
2024-12-03T01:15:42,408 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49 to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/41de361a84f44cdc9cdb88b4166c1c49
2024-12-03T01:15:42,410 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0 to hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/info/8e6e5fc41713440ca9041c66925a1cd0
2024-12-03T01:15:42,416 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/c01e28d9af22e6a31ddf3338064570cf/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-03T01:15:42,416 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:42,416 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c01e28d9af22e6a31ddf3338064570cf:

2024-12-03T01:15:42,417 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733188482240.c01e28d9af22e6a31ddf3338064570cf.
2024-12-03T01:15:42,417 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0d06b76a38c879ff50377898eba23cc6, disabling compactions & flushes
2024-12-03T01:15:42,417 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:15:42,417 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:15:42,417 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6. after waiting 0 ms
2024-12-03T01:15:42,417 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:15:42,420 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/namespace/0d06b76a38c879ff50377898eba23cc6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T01:15:42,421 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:15:42,421 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0d06b76a38c879ff50377898eba23cc6:

2024-12-03T01:15:42,421 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733188481726.0d06b76a38c879ff50377898eba23cc6.
2024-12-03T01:15:42,423 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/table/5dedf2ad9ee545d58b14794b7dd7c4d0 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733188483028/Put/seqid=0
2024-12-03T01:15:42,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741850_1026 (size=5532)
2024-12-03T01:15:42,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741850_1026 (size=5532)
2024-12-03T01:15:42,428 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=264 B at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/table/5dedf2ad9ee545d58b14794b7dd7c4d0
2024-12-03T01:15:42,434 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/info/e04d5bbdcf074c93b98a3ab035da9844 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/info/e04d5bbdcf074c93b98a3ab035da9844
2024-12-03T01:15:42,439 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/info/e04d5bbdcf074c93b98a3ab035da9844, entries=20, sequenceid=14, filesize=8.2 K
2024-12-03T01:15:42,440 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/.tmp/table/5dedf2ad9ee545d58b14794b7dd7c4d0 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/table/5dedf2ad9ee545d58b14794b7dd7c4d0
2024-12-03T01:15:42,446 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/table/5dedf2ad9ee545d58b14794b7dd7c4d0, entries=4, sequenceid=14, filesize=5.4 K
2024-12-03T01:15:42,447 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~3.05 KB/3122, heapSize ~5.27 KB/5400, currentSize=0 B/0 for 1588230740 in 71ms, sequenceid=14, compaction requested=false
2024-12-03T01:15:42,457 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=1
2024-12-03T01:15:42,458 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T01:15:42,458 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:15:42,458 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:15:42,458 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T01:15:42,576 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,44179,1733188480704; all regions closed.
2024-12-03T01:15:42,576 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:42,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741834_1010 (size=4570)
2024-12-03T01:15:42,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741834_1010 (size=4570)
2024-12-03T01:15:42,582 DEBUG [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs
2024-12-03T01:15:42,582 INFO  [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C44179%2C1733188480704.meta:.meta(num 1733188481675)
2024-12-03T01:15:42,582 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/WALs/1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:42,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741847_1023 (size=1545)
2024-12-03T01:15:42,584 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741847_1023 (size=1545)
2024-12-03T01:15:42,588 DEBUG [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(1071): Moved 2 WAL file(s) to /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/oldWALs
2024-12-03T01:15:42,588 INFO  [RS:0;1a7b1a4db65e:44179 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C44179%2C1733188480704:(num 1733188542363)
2024-12-03T01:15:42,588 DEBUG [RS:0;1a7b1a4db65e:44179 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:15:42,588 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:15:42,589 INFO  [RS:0;1a7b1a4db65e:44179 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-03T01:15:42,589 INFO  [RS:0;1a7b1a4db65e:44179 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44179
2024-12-03T01:15:42,590 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:15:42,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,44179,1733188480704
2024-12-03T01:15:42,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:15:42,594 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,44179,1733188480704]
2024-12-03T01:15:42,594 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,44179,1733188480704; numProcessing=1
2024-12-03T01:15:42,596 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,44179,1733188480704 already deleted, retry=false
2024-12-03T01:15:42,596 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,44179,1733188480704 expired; onlineServers=0
2024-12-03T01:15:42,596 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,42773,1733188480639' *****
2024-12-03T01:15:42,596 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:15:42,596 DEBUG [M:0;1a7b1a4db65e:42773 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca7dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:15:42,596 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,42773,1733188480639
2024-12-03T01:15:42,596 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,42773,1733188480639; all regions closed.
2024-12-03T01:15:42,596 DEBUG [M:0;1a7b1a4db65e:42773 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:15:42,597 DEBUG [M:0;1a7b1a4db65e:42773 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:15:42,597 DEBUG [M:0;1a7b1a4db65e:42773 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:15:42,597 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:15:42,597 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188480897 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188480897,5,FailOnTimeoutGroup]
2024-12-03T01:15:42,597 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188480898 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188480898,5,FailOnTimeoutGroup]
2024-12-03T01:15:42,597 INFO  [M:0;1a7b1a4db65e:42773 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:15:42,597 DEBUG [M:0;1a7b1a4db65e:42773 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:15:42,597 INFO  [M:0;1a7b1a4db65e:42773 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:15:42,597 INFO  [M:0;1a7b1a4db65e:42773 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:15:42,598 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:15:42,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:15:42,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:42,599 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:15:42,599 DEBUG [M:0;1a7b1a4db65e:42773 {}] zookeeper.ZKUtil(347): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:15:42,599 WARN  [M:0;1a7b1a4db65e:42773 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:15:42,599 INFO  [M:0;1a7b1a4db65e:42773 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:15:42,599 INFO  [M:0;1a7b1a4db65e:42773 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:15:42,599 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:15:42,599 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:42,599 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:42,599 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:15:42,599 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:42,599 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=65.09 KB heapSize=81.71 KB
2024-12-03T01:15:42,624 DEBUG [M:0;1a7b1a4db65e:42773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/acc2b13fae9341419641d2fcdc2248c4 is 82, key is hbase:meta,,1/info:regioninfo/1733188481701/Put/seqid=0
2024-12-03T01:15:42,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741851_1027 (size=5672)
2024-12-03T01:15:42,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741851_1027 (size=5672)
2024-12-03T01:15:42,635 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/acc2b13fae9341419641d2fcdc2248c4
2024-12-03T01:15:42,658 DEBUG [M:0;1a7b1a4db65e:42773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c78a423e273446bbfd9b4753944a3ad is 799, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733188483036/Put/seqid=0
2024-12-03T01:15:42,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741852_1028 (size=8357)
2024-12-03T01:15:42,670 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741852_1028 (size=8357)
2024-12-03T01:15:42,674 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.48 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c78a423e273446bbfd9b4753944a3ad
2024-12-03T01:15:42,679 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2c78a423e273446bbfd9b4753944a3ad
2024-12-03T01:15:42,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:15:42,695 INFO  [RS:0;1a7b1a4db65e:44179 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,44179,1733188480704; zookeeper connection closed.
2024-12-03T01:15:42,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44179-0x10175e357650001, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:15:42,695 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b231abe {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b231abe
2024-12-03T01:15:42,695 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T01:15:42,697 DEBUG [M:0;1a7b1a4db65e:42773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a175875254d04ce59949f4e86a0eaf31 is 69, key is 1a7b1a4db65e,44179,1733188480704/rs:state/1733188480940/Put/seqid=0
2024-12-03T01:15:42,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741853_1029 (size=5156)
2024-12-03T01:15:42,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741853_1029 (size=5156)
2024-12-03T01:15:42,707 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a175875254d04ce59949f4e86a0eaf31
2024-12-03T01:15:42,736 DEBUG [M:0;1a7b1a4db65e:42773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1aa527216fe04283b9404d848b0e3aa5 is 52, key is load_balancer_on/state:d/1733188482234/Put/seqid=0
2024-12-03T01:15:42,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741854_1030 (size=5056)
2024-12-03T01:15:42,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741854_1030 (size=5056)
2024-12-03T01:15:42,745 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1aa527216fe04283b9404d848b0e3aa5
2024-12-03T01:15:42,755 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/acc2b13fae9341419641d2fcdc2248c4 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/acc2b13fae9341419641d2fcdc2248c4
2024-12-03T01:15:42,763 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/acc2b13fae9341419641d2fcdc2248c4, entries=8, sequenceid=184, filesize=5.5 K
2024-12-03T01:15:42,764 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2c78a423e273446bbfd9b4753944a3ad as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2c78a423e273446bbfd9b4753944a3ad
2024-12-03T01:15:42,789 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2c78a423e273446bbfd9b4753944a3ad
2024-12-03T01:15:42,790 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2c78a423e273446bbfd9b4753944a3ad, entries=21, sequenceid=184, filesize=8.2 K
2024-12-03T01:15:42,791 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a175875254d04ce59949f4e86a0eaf31 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a175875254d04ce59949f4e86a0eaf31
2024-12-03T01:15:42,797 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a175875254d04ce59949f4e86a0eaf31, entries=1, sequenceid=184, filesize=5.0 K
2024-12-03T01:15:42,798 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1aa527216fe04283b9404d848b0e3aa5 as hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1aa527216fe04283b9404d848b0e3aa5
2024-12-03T01:15:42,803 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:33065/user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1aa527216fe04283b9404d848b0e3aa5, entries=1, sequenceid=184, filesize=4.9 K
2024-12-03T01:15:42,804 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(3040): Finished flush of dataSize ~65.09 KB/66649, heapSize ~81.65 KB/83608, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 205ms, sequenceid=184, compaction requested=false
2024-12-03T01:15:42,806 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:42,806 DEBUG [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:15:42,806 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/2acb5645-6a09-0dc2-7ed3-ddcbdef760d9/MasterData/WALs/1a7b1a4db65e,42773,1733188480639
2024-12-03T01:15:42,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44957 is added to blk_1073741830_1006 (size=79170)
2024-12-03T01:15:42,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33189 is added to blk_1073741830_1006 (size=79170)
2024-12-03T01:15:42,809 INFO  [M:0;1a7b1a4db65e:42773 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:15:42,809 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:15:42,809 INFO  [M:0;1a7b1a4db65e:42773 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:42773
2024-12-03T01:15:42,811 DEBUG [M:0;1a7b1a4db65e:42773 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,42773,1733188480639 already deleted, retry=false
2024-12-03T01:15:42,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:15:42,912 INFO  [M:0;1a7b1a4db65e:42773 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,42773,1733188480639; zookeeper connection closed.
2024-12-03T01:15:42,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42773-0x10175e357650000, quorum=127.0.0.1:63635, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:15:42,915 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea2a712{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:15:42,916 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3fa2966d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:15:42,916 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:15:42,916 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c7be892{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:15:42,916 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59bc5232{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,STOPPED}
2024-12-03T01:15:42,918 WARN  [BP-345640683-172.17.0.2-1733188479059 heartbeating to localhost/127.0.0.1:33065 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:15:42,918 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:15:42,918 WARN  [BP-345640683-172.17.0.2-1733188479059 heartbeating to localhost/127.0.0.1:33065 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-345640683-172.17.0.2-1733188479059 (Datanode Uuid 6fa99f82-1612-4180-9130-7d238f5a8c2d) service to localhost/127.0.0.1:33065
2024-12-03T01:15:42,918 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:15:42,919 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data3/current/BP-345640683-172.17.0.2-1733188479059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:15:42,919 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data4/current/BP-345640683-172.17.0.2-1733188479059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:15:42,919 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:15:42,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4fe74ba6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:15:42,922 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@402c6ca9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:15:42,922 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:15:42,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75e97d20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:15:42,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@903456d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,STOPPED}
2024-12-03T01:15:42,924 WARN  [BP-345640683-172.17.0.2-1733188479059 heartbeating to localhost/127.0.0.1:33065 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:15:42,924 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:15:42,924 WARN  [BP-345640683-172.17.0.2-1733188479059 heartbeating to localhost/127.0.0.1:33065 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-345640683-172.17.0.2-1733188479059 (Datanode Uuid 4f940019-7dd2-419c-8f04-7e3802432550) service to localhost/127.0.0.1:33065
2024-12-03T01:15:42,924 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:15:42,925 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data1/current/BP-345640683-172.17.0.2-1733188479059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:15:42,925 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/cluster_f8f16462-4981-114e-36c7-4214519cd870/dfs/data/data2/current/BP-345640683-172.17.0.2-1733188479059 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:15:42,925 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:15:42,931 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@649a505c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:15:42,932 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6ec653a7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:15:42,932 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:15:42,932 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f61a378{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:15:42,932 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e4e700f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir/,STOPPED}
2024-12-03T01:15:42,942 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:15:42,962 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:15:42,965 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:15:42,975 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=108 (was 101) - Thread LEAK? -, OpenFileDescriptor=464 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 276) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2579 (was 3713)
2024-12-03T01:15:42,985 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=109, OpenFileDescriptor=464, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=2579
2024-12-03T01:15:42,985 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:15:42,985 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.log.dir so I do NOT create it in target/test-data/dfcee134-6552-e340-ec24-494b426c30f2
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/1354e44f-42b7-ed7f-03e9-61400e0a029c/hadoop.tmp.dir so I do NOT create it in target/test-data/dfcee134-6552-e340-ec24-494b426c30f2
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47, deleteOnExit=true
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/test.cache.data in system properties and HBase conf
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:15:42,986 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:15:42,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:15:42,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:15:42,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:15:43,009 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:15:43,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:43,102 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:15:43,111 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:15:43,116 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:15:43,116 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:15:43,116 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:15:43,117 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:15:43,122 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58c48638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:15:43,122 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e2b1db8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:15:43,287 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@72ecbbb0{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/java.io.tmpdir/jetty-localhost-38219-hadoop-hdfs-3_4_1-tests_jar-_-any-5397310193852406894/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:15:43,288 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11832530{HTTP/1.1, (http/1.1)}{localhost:38219}
2024-12-03T01:15:43,288 INFO  [Time-limited test {}] server.Server(415): Started @291287ms
2024-12-03T01:15:43,309 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:15:43,468 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:15:43,473 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:15:43,493 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:15:43,493 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:15:43,493 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:15:43,501 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f3fedd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:15:43,501 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1104cb6c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:15:43,642 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@734fa1f2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/java.io.tmpdir/jetty-localhost-46627-hadoop-hdfs-3_4_1-tests_jar-_-any-1016794248928696941/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:15:43,642 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@24316e4c{HTTP/1.1, (http/1.1)}{localhost:46627}
2024-12-03T01:15:43,642 INFO  [Time-limited test {}] server.Server(415): Started @291642ms
2024-12-03T01:15:43,644 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:15:43,706 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:15:43,713 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:15:43,725 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:15:43,725 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:15:43,725 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:15:43,726 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@533ed053{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:15:43,726 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3daa8d32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:15:43,774 WARN  [Thread-1695 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data2/current/BP-1275478181-172.17.0.2-1733188543022/current, will proceed with Du for space computation calculation, 
2024-12-03T01:15:43,774 WARN  [Thread-1694 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data1/current/BP-1275478181-172.17.0.2-1733188543022/current, will proceed with Du for space computation calculation, 
2024-12-03T01:15:43,799 WARN  [Thread-1673 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:15:43,802 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb276ded9887f0bb4 with lease ID 0x3d82f9730ee81e75: Processing first storage report for DS-ce39cd63-4974-4eda-af03-1591c5e00fc2 from datanode DatanodeRegistration(127.0.0.1:46169, datanodeUuid=bc866916-f249-411f-a0c9-82690b8688f2, infoPort=33535, infoSecurePort=0, ipcPort=45843, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022)
2024-12-03T01:15:43,802 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb276ded9887f0bb4 with lease ID 0x3d82f9730ee81e75: from storage DS-ce39cd63-4974-4eda-af03-1591c5e00fc2 node DatanodeRegistration(127.0.0.1:46169, datanodeUuid=bc866916-f249-411f-a0c9-82690b8688f2, infoPort=33535, infoSecurePort=0, ipcPort=45843, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:15:43,802 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb276ded9887f0bb4 with lease ID 0x3d82f9730ee81e75: Processing first storage report for DS-4dfa7d3e-3492-429a-bdbe-208a6d3df881 from datanode DatanodeRegistration(127.0.0.1:46169, datanodeUuid=bc866916-f249-411f-a0c9-82690b8688f2, infoPort=33535, infoSecurePort=0, ipcPort=45843, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022)
2024-12-03T01:15:43,802 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb276ded9887f0bb4 with lease ID 0x3d82f9730ee81e75: from storage DS-4dfa7d3e-3492-429a-bdbe-208a6d3df881 node DatanodeRegistration(127.0.0.1:46169, datanodeUuid=bc866916-f249-411f-a0c9-82690b8688f2, infoPort=33535, infoSecurePort=0, ipcPort=45843, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:15:43,855 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9e1a806{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/java.io.tmpdir/jetty-localhost-41583-hadoop-hdfs-3_4_1-tests_jar-_-any-10965827471348440502/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:15:43,855 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b90b736{HTTP/1.1, (http/1.1)}{localhost:41583}
2024-12-03T01:15:43,855 INFO  [Time-limited test {}] server.Server(415): Started @291855ms
2024-12-03T01:15:43,857 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:15:43,989 WARN  [Thread-1720 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data3/current/BP-1275478181-172.17.0.2-1733188543022/current, will proceed with Du for space computation calculation, 
2024-12-03T01:15:43,990 WARN  [Thread-1721 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data4/current/BP-1275478181-172.17.0.2-1733188543022/current, will proceed with Du for space computation calculation, 
2024-12-03T01:15:44,022 WARN  [Thread-1709 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:15:44,029 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0c9f29e652daeb1 with lease ID 0x3d82f9730ee81e76: Processing first storage report for DS-bfe0322a-3728-401b-b78c-d00dbd1c59a8 from datanode DatanodeRegistration(127.0.0.1:35921, datanodeUuid=449a28b7-b637-4546-a1db-fe493480ce77, infoPort=45751, infoSecurePort=0, ipcPort=43775, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022)
2024-12-03T01:15:44,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0c9f29e652daeb1 with lease ID 0x3d82f9730ee81e76: from storage DS-bfe0322a-3728-401b-b78c-d00dbd1c59a8 node DatanodeRegistration(127.0.0.1:35921, datanodeUuid=449a28b7-b637-4546-a1db-fe493480ce77, infoPort=45751, infoSecurePort=0, ipcPort=43775, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:15:44,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc0c9f29e652daeb1 with lease ID 0x3d82f9730ee81e76: Processing first storage report for DS-251a0b70-1868-4a6e-8fd7-f539b063b1b8 from datanode DatanodeRegistration(127.0.0.1:35921, datanodeUuid=449a28b7-b637-4546-a1db-fe493480ce77, infoPort=45751, infoSecurePort=0, ipcPort=43775, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022)
2024-12-03T01:15:44,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc0c9f29e652daeb1 with lease ID 0x3d82f9730ee81e76: from storage DS-251a0b70-1868-4a6e-8fd7-f539b063b1b8 node DatanodeRegistration(127.0.0.1:35921, datanodeUuid=449a28b7-b637-4546-a1db-fe493480ce77, infoPort=45751, infoSecurePort=0, ipcPort=43775, storageInfo=lv=-57;cid=testClusterID;nsid=1858372846;c=1733188543022), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:15:44,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:44,097 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2
2024-12-03T01:15:44,117 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/zookeeper_0, clientPort=65274, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:15:44,125 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=65274
2024-12-03T01:15:44,125 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,127 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:15:44,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:15:44,170 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e with version=8
2024-12-03T01:15:44,170 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:15:44,173 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:15:44,174 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:15:44,195 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:44017
2024-12-03T01:15:44,196 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,198 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,201 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44017 connecting to ZooKeeper ensemble=127.0.0.1:65274
2024-12-03T01:15:44,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:440170x0, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:15:44,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44017-0x10175e450f80000 connected
2024-12-03T01:15:44,246 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:15:44,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:15:44,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:15:44,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44017
2024-12-03T01:15:44,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44017
2024-12-03T01:15:44,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44017
2024-12-03T01:15:44,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44017
2024-12-03T01:15:44,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44017
2024-12-03T01:15:44,260 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e, hbase.cluster.distributed=false
2024-12-03T01:15:44,276 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:15:44,276 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:15:44,277 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:15:44,278 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45383
2024-12-03T01:15:44,279 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:15:44,279 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:15:44,280 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,282 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,284 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45383 connecting to ZooKeeper ensemble=127.0.0.1:65274
2024-12-03T01:15:44,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453830x0, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:15:44,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453830x0, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:15:44,287 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45383-0x10175e450f80001 connected
2024-12-03T01:15:44,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:15:44,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:15:44,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45383
2024-12-03T01:15:44,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45383
2024-12-03T01:15:44,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45383
2024-12-03T01:15:44,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45383
2024-12-03T01:15:44,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45383
2024-12-03T01:15:44,290 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:15:44,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:15:44,292 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:15:44,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:15:44,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,294 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:15:44,294 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,44017,1733188544172 from backup master directory
2024-12-03T01:15:44,294 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:15:44,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:15:44,297 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:15:44,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:15:44,297 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,303 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:44017
2024-12-03T01:15:44,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:15:44,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:15:44,310 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/hbase.id with ID: 83bc117e-bed9-449d-83c7-22eb404435f4
2024-12-03T01:15:44,321 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:44,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:15:44,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:15:44,334 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:15:44,334 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:15:44,335 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:15:44,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:15:44,346 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:15:44,349 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store
2024-12-03T01:15:44,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:15:44,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:15:44,367 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:44,367 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:15:44,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:15:44,368 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/.initializing
2024-12-03T01:15:44,368 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/WALs/1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,373 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C44017%2C1733188544172, suffix=, logDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/WALs/1a7b1a4db65e,44017,1733188544172, archiveDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/oldWALs, maxLogs=10
2024-12-03T01:15:44,374 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C44017%2C1733188544172.1733188544374
2024-12-03T01:15:44,395 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/WALs/1a7b1a4db65e,44017,1733188544172/1a7b1a4db65e%2C44017%2C1733188544172.1733188544374
2024-12-03T01:15:44,395 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45751:45751),(127.0.0.1/127.0.0.1:33535:33535)]
2024-12-03T01:15:44,395 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:15:44,395 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:44,395 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:15:44,395 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:15:44,413 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:15:44,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:15:44,415 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:44,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:15:44,417 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:15:44,418 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:15:44,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:15:44,420 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:15:44,420 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:15:44,421 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:15:44,429 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:15:44,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,436 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:15:44,441 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:15:44,442 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:15:44,444 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:15:44,445 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:15:44,447 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:15:44,448 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=725023, jitterRate=-0.07808589935302734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:15:44,449 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:15:44,449 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:15:44,452 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccd1226, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:15:44,453 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:15:44,453 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:15:44,453 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:15:44,453 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:15:44,454 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T01:15:44,454 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-03T01:15:44,454 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:15:44,456 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:15:44,457 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:15:44,458 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:15:44,458 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:15:44,459 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:15:44,460 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:15:44,460 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:15:44,461 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:15:44,463 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:15:44,464 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:15:44,465 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:15:44,467 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:15:44,468 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:15:44,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:15:44,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:15:44,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,470 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,44017,1733188544172, sessionid=0x10175e450f80000, setting cluster-up flag (Was=false)
2024-12-03T01:15:44,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,480 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:15:44,481 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:44,490 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:15:44,490 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:44,493 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:15:44,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:15:44,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:15:44,493 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,44017,1733188544172
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:15:44,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188574497
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:15:44,497 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:15:44,497 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:15:44,497 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:15:44,498 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:15:44,499 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,499 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:15:44,499 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188544498,5,FailOnTimeoutGroup]
2024-12-03T01:15:44,499 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188544499,5,FailOnTimeoutGroup]
2024-12-03T01:15:44,499 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,500 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:15:44,500 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,500 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:15:44,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:15:44,509 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:15:44,509 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e
2024-12-03T01:15:44,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:15:44,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:15:44,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:44,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:15:44,569 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:15:44,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,570 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:44,570 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:15:44,571 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:15:44,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,572 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:44,572 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:15:44,573 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:15:44,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:44,574 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:44,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740
2024-12-03T01:15:44,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740
2024-12-03T01:15:44,580 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:15:44,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:15:44,585 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705689, jitterRate=-0.10267050564289093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:15:44,585 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:15:44,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:15:44,588 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:15:44,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:15:44,612 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:45383
2024-12-03T01:15:44,613 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:15:44,613 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:15:44,613 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:15:44,617 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1008): ClusterId : 83bc117e-bed9-449d-83c7-22eb404435f4
2024-12-03T01:15:44,617 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:15:44,620 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:15:44,624 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:15:44,629 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:15:44,629 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:15:44,633 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:15:44,633 DEBUG [RS:0;1a7b1a4db65e:45383 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a49deb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:15:44,636 DEBUG [RS:0;1a7b1a4db65e:45383 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417297ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:15:44,637 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:15:44,637 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:15:44,637 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:15:44,637 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,44017,1733188544172 with isa=1a7b1a4db65e/172.17.0.2:45383, startcode=1733188544276
2024-12-03T01:15:44,637 DEBUG [RS:0;1a7b1a4db65e:45383 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:15:44,644 INFO  [RS-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56605, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:15:44,644 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44017 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,644 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44017 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,646 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e
2024-12-03T01:15:44,646 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:34085
2024-12-03T01:15:44,646 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:15:44,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:15:44,653 DEBUG [RS:0;1a7b1a4db65e:45383 {}] zookeeper.ZKUtil(111): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,653 WARN  [RS:0;1a7b1a4db65e:45383 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:15:44,653 INFO  [RS:0;1a7b1a4db65e:45383 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:15:44,653 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,654 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,45383,1733188544276]
2024-12-03T01:15:44,657 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:15:44,657 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:15:44,659 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:15:44,659 INFO  [RS:0;1a7b1a4db65e:45383 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:15:44,659 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,660 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:15:44,664 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:15:44,664 DEBUG [RS:0;1a7b1a4db65e:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:15:44,665 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,665 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,665 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,665 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,665 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,45383,1733188544276-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:15:44,684 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:15:44,684 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,45383,1733188544276-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:44,699 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.Replication(204): 1a7b1a4db65e,45383,1733188544276 started
2024-12-03T01:15:44,699 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,45383,1733188544276, RpcServer on 1a7b1a4db65e/172.17.0.2:45383, sessionid=0x10175e450f80001
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,45383,1733188544276'
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:15:44,699 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,45383,1733188544276'
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:15:44,700 DEBUG [RS:0;1a7b1a4db65e:45383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:15:44,700 INFO  [RS:0;1a7b1a4db65e:45383 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:15:44,700 INFO  [RS:0;1a7b1a4db65e:45383 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:15:44,775 WARN  [1a7b1a4db65e:44017 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-03T01:15:44,802 INFO  [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C45383%2C1733188544276, suffix=, logDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276, archiveDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs, maxLogs=32
2024-12-03T01:15:44,803 INFO  [RS:0;1a7b1a4db65e:45383 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C45383%2C1733188544276.1733188544803
2024-12-03T01:15:44,809 INFO  [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188544803
2024-12-03T01:15:44,809 DEBUG [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33535:33535),(127.0.0.1/127.0.0.1:45751:45751)]
2024-12-03T01:15:45,025 DEBUG [1a7b1a4db65e:44017 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:15:45,025 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,026 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,45383,1733188544276, state=OPENING
2024-12-03T01:15:45,028 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:15:45,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:45,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:45,030 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:15:45,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:15:45,030 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:15:45,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:45,182 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,183 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:15:45,185 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51966, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:15:45,188 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:15:45,188 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:15:45,190 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C45383%2C1733188544276.meta, suffix=.meta, logDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276, archiveDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs, maxLogs=32
2024-12-03T01:15:45,191 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C45383%2C1733188544276.meta.1733188545190.meta
2024-12-03T01:15:45,196 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.meta.1733188545190.meta
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45751:45751),(127.0.0.1/127.0.0.1:33535:33535)]
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:15:45,196 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:15:45,196 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:45,197 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:15:45,197 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:15:45,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:15:45,198 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:15:45,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,199 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:45,199 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:15:45,200 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:15:45,200 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,200 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:45,200 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:15:45,201 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:15:45,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,202 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:15:45,202 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740
2024-12-03T01:15:45,204 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740
2024-12-03T01:15:45,205 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:15:45,207 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:15:45,208 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702716, jitterRate=-0.10645096004009247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:15:45,208 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:15:45,209 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188545182
2024-12-03T01:15:45,210 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:15:45,210 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:15:45,211 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,211 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,45383,1733188544276, state=OPEN
2024-12-03T01:15:45,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:15:45,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:15:45,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:15:45,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:15:45,219 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:15:45,219 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,45383,1733188544276 in 187 msec
2024-12-03T01:15:45,221 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:15:45,221 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec
2024-12-03T01:15:45,222 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 729 msec
2024-12-03T01:15:45,222 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188545222, completionTime=-1
2024-12-03T01:15:45,222 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:15:45,223 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:15:45,223 DEBUG [hconnection-0x4573948b-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:15:45,225 INFO  [RS-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51972, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:15:45,226 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:15:45,226 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188605226
2024-12-03T01:15:45,226 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188665226
2024-12-03T01:15:45,226 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 3 msec
2024-12-03T01:15:45,232 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,232 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,232 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,232 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:44017, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,232 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,233 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:15:45,233 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:15:45,234 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:15:45,234 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:15:45,235 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:15:45,235 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,236 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:15:45,242 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:15:45,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:15:45,244 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 97f43709f8e893c79b431e7315d66f1f, NAME => 'hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e
2024-12-03T01:15:45,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:15:45,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 97f43709f8e893c79b431e7315d66f1f, disabling compactions & flushes
2024-12-03T01:15:45,252 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f. after waiting 0 ms
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,252 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,252 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 97f43709f8e893c79b431e7315d66f1f:

2024-12-03T01:15:45,253 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:15:45,254 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188545253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188545253"}]},"ts":"1733188545253"}
2024-12-03T01:15:45,255 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:15:45,256 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:15:45,256 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188545256"}]},"ts":"1733188545256"}
2024-12-03T01:15:45,258 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:15:45,262 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=97f43709f8e893c79b431e7315d66f1f, ASSIGN}]
2024-12-03T01:15:45,263 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=97f43709f8e893c79b431e7315d66f1f, ASSIGN
2024-12-03T01:15:45,264 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=97f43709f8e893c79b431e7315d66f1f, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,45383,1733188544276; forceNewPlan=false, retain=false
2024-12-03T01:15:45,414 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=97f43709f8e893c79b431e7315d66f1f, regionState=OPENING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,417 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 97f43709f8e893c79b431e7315d66f1f, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:15:45,570 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,575 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 97f43709f8e893c79b431e7315d66f1f, NAME => 'hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:15:45,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:45,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,577 INFO  [StoreOpener-97f43709f8e893c79b431e7315d66f1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 97f43709f8e893c79b431e7315d66f1f 
2024-12-03T01:15:45,579 INFO  [StoreOpener-97f43709f8e893c79b431e7315d66f1f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97f43709f8e893c79b431e7315d66f1f columnFamilyName info
2024-12-03T01:15:45,579 DEBUG [StoreOpener-97f43709f8e893c79b431e7315d66f1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,579 INFO  [StoreOpener-97f43709f8e893c79b431e7315d66f1f-1 {}] regionserver.HStore(327): Store=97f43709f8e893c79b431e7315d66f1f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:15:45,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:15:45,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:15:45,586 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 97f43709f8e893c79b431e7315d66f1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=697242, jitterRate=-0.11341109871864319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:15:45,586 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 97f43709f8e893c79b431e7315d66f1f:

2024-12-03T01:15:45,587 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f., pid=6, masterSystemTime=1733188545570
2024-12-03T01:15:45,592 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,592 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:15:45,593 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=97f43709f8e893c79b431e7315d66f1f, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,599 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:15:45,599 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 97f43709f8e893c79b431e7315d66f1f, server=1a7b1a4db65e,45383,1733188544276 in 178 msec
2024-12-03T01:15:45,601 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:15:45,601 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=97f43709f8e893c79b431e7315d66f1f, ASSIGN in 337 msec
2024-12-03T01:15:45,602 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:15:45,602 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188545602"}]},"ts":"1733188545602"}
2024-12-03T01:15:45,604 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:15:45,611 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:15:45,613 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 378 msec
2024-12-03T01:15:45,635 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:15:45,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:45,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:15:45,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:15:45,641 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:15:45,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:15:45,659 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 17 msec
2024-12-03T01:15:45,663 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:15:45,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:15:45,675 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 11 msec
2024-12-03T01:15:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:15:45,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.395sec
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:15:45,692 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:15:45,696 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x400591a6 to 127.0.0.1:65274 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@448d19e6
2024-12-03T01:15:45,696 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:15:45,696 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:15:45,697 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,44017,1733188544172-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:15:45,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a89306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:15:45,703 DEBUG [hconnection-0x89a4151-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:15:45,706 INFO  [RS-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51986, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:15:45,709 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,44017,1733188544172
2024-12-03T01:15:45,710 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:15:45,712 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:15:45,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-03T01:15:45,721 INFO  [RS-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42436, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-03T01:15:45,722 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-03T01:15:45,722 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-03T01:15:45,723 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:15:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-03T01:15:45,725 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:15:45,725 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:45,725 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 9
2024-12-03T01:15:45,726 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:15:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:15:45,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741837_1013 (size=381)
2024-12-03T01:15:45,739 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741837_1013 (size=381)
2024-12-03T01:15:45,741 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 57504ad29584e935de6eb6630505b158, NAME => 'TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e
2024-12-03T01:15:45,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741838_1014 (size=64)
2024-12-03T01:15:45,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741838_1014 (size=64)
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1681): Closing 57504ad29584e935de6eb6630505b158, disabling compactions & flushes
2024-12-03T01:15:45,782 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158. after waiting 0 ms
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:45,782 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:45,782 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1635): Region close journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:15:45,784 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:15:45,784 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733188545784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188545784"}]},"ts":"1733188545784"}
2024-12-03T01:15:45,789 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:15:45,790 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:15:45,790 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188545790"}]},"ts":"1733188545790"}
2024-12-03T01:15:45,792 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-03T01:15:45,797 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, ASSIGN}]
2024-12-03T01:15:45,799 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, ASSIGN
2024-12-03T01:15:45,806 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,45383,1733188544276; forceNewPlan=false, retain=false
2024-12-03T01:15:45,956 INFO  [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=57504ad29584e935de6eb6630505b158, regionState=OPENING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:45,959 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:15:46,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:46,110 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:46,114 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:46,115 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => 57504ad29584e935de6eb6630505b158, NAME => 'TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:15:46,115 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,115 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:15:46,115 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for 57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,115 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for 57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,124 INFO  [StoreOpener-57504ad29584e935de6eb6630505b158-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 57504ad29584e935de6eb6630505b158 
2024-12-03T01:15:46,128 INFO  [StoreOpener-57504ad29584e935de6eb6630505b158-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57504ad29584e935de6eb6630505b158 columnFamilyName info
2024-12-03T01:15:46,128 DEBUG [StoreOpener-57504ad29584e935de6eb6630505b158-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:15:46,128 INFO  [StoreOpener-57504ad29584e935de6eb6630505b158-1 {}] regionserver.HStore(327): Store=57504ad29584e935de6eb6630505b158/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:15:46,129 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,131 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,134 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for 57504ad29584e935de6eb6630505b158
2024-12-03T01:15:46,141 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:15:46,142 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened 57504ad29584e935de6eb6630505b158; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=716861, jitterRate=-0.08846446871757507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:15:46,142 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:15:46,143 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158., pid=11, masterSystemTime=1733188546110
2024-12-03T01:15:46,145 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:46,145 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:46,145 INFO  [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=57504ad29584e935de6eb6630505b158, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:46,149 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10
2024-12-03T01:15:46,150 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276 in 188 msec
2024-12-03T01:15:46,151 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9
2024-12-03T01:15:46,151 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, ASSIGN in 352 msec
2024-12-03T01:15:46,152 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:15:46,152 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188546152"}]},"ts":"1733188546152"}
2024-12-03T01:15:46,154 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-03T01:15:46,157 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:15:46,159 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestLogRolling-testLogRolling in 434 msec
2024-12-03T01:15:47,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:47,417 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,418 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,440 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,440 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,440 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,441 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,441 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,447 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,447 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,447 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,450 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,959 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:15:47,960 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,960 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,961 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,961 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,961 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,962 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,996 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:47,997 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:48,000 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:48,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:48,331 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-03T01:15:48,332 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-03T01:15:48,333 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-03T01:15:49,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:50,041 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:50,657 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-03T01:15:50,658 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace'
2024-12-03T01:15:50,658 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-03T01:15:51,041 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:52,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:53,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:53,835 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:15:53,836 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,838 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,866 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,867 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,867 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,868 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:53,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:15:54,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:54,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 23930
2024-12-03T01:15:55,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44017 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9
2024-12-03T01:15:55,728 INFO  [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling, procId: 9 completed
2024-12-03T01:15:55,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2718): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-03T01:15:55,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(2724): firstRegionName=TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:15:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on 57504ad29584e935de6eb6630505b158
2024-12-03T01:15:55,749 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:15:55,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/7c5f3b52a0a248db967538de89525f98 is 1080, key is row0001/info:/1733188555734/Put/seqid=0
2024-12-03T01:15:55,817 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741839_1015 (size=12509)
2024-12-03T01:15:55,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741839_1015 (size=12509)
2024-12-03T01:15:55,822 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:15:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188565821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:15:55,835 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/7c5f3b52a0a248db967538de89525f98
2024-12-03T01:15:55,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/7c5f3b52a0a248db967538de89525f98 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98
2024-12-03T01:15:55,858 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98, entries=7, sequenceid=11, filesize=12.2 K
2024-12-03T01:15:55,865 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 57504ad29584e935de6eb6630505b158 in 115ms, sequenceid=11, compaction requested=false
2024-12-03T01:15:55,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:15:56,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:57,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:58,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:15:58,675 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=1, created chunk count=15, reused chunk count=36, reuseRatio=70.59%
2024-12-03T01:15:58,675 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-03T01:15:59,047 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:00,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:01,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:02,049 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:03,050 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:04,050 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:05,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:05,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,343 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:05,878 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:16:05,883 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:16:05,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/3b86c4e7e37f49b4a9caf6d8ac7431a3 is 1080, key is row0008/info:/1733188555751/Put/seqid=0
2024-12-03T01:16:05,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741840_1016 (size=29761)
2024-12-03T01:16:05,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741840_1016 (size=29761)
2024-12-03T01:16:05,904 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/3b86c4e7e37f49b4a9caf6d8ac7431a3
2024-12-03T01:16:05,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,920 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/3b86c4e7e37f49b4a9caf6d8ac7431a3 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3
2024-12-03T01:16:05,921 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,921 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:05,934 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3, entries=23, sequenceid=37, filesize=29.1 K
2024-12-03T01:16:05,936 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 57504ad29584e935de6eb6630505b158 in 58ms, sequenceid=37, compaction requested=false
2024-12-03T01:16:05,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:05,936 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=41.3 K, sizeToCheck=16.0 K
2024-12-03T01:16:05,936 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:05,937 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3 because midkey is the same as first or last row
2024-12-03T01:16:06,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:07,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:07,889 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:16:07,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/9df512126e7143a4b5f4bc33d2f480e3 is 1080, key is row0031/info:/1733188565878/Put/seqid=0
2024-12-03T01:16:07,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741841_1017 (size=12509)
2024-12-03T01:16:07,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741841_1017 (size=12509)
2024-12-03T01:16:07,919 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:16:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188577918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:08,053 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:08,308 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/9df512126e7143a4b5f4bc33d2f480e3
2024-12-03T01:16:08,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/9df512126e7143a4b5f4bc33d2f480e3 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3
2024-12-03T01:16:08,321 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3, entries=7, sequenceid=47, filesize=12.2 K
2024-12-03T01:16:08,322 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 57504ad29584e935de6eb6630505b158 in 433ms, sequenceid=47, compaction requested=true
2024-12-03T01:16:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=53.5 K, sizeToCheck=16.0 K
2024-12-03T01:16:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:08,322 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3 because midkey is the same as first or last row
2024-12-03T01:16:08,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57504ad29584e935de6eb6630505b158:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:16:08,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:08,323 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:16:08,324 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:16:08,324 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): 57504ad29584e935de6eb6630505b158/info is initiating minor compaction (all files)
2024-12-03T01:16:08,324 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 57504ad29584e935de6eb6630505b158/info in TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:08,324 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp, totalSize=53.5 K
2024-12-03T01:16:08,325 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7c5f3b52a0a248db967538de89525f98, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733188555734
2024-12-03T01:16:08,325 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3b86c4e7e37f49b4a9caf6d8ac7431a3, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733188555751
2024-12-03T01:16:08,325 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9df512126e7143a4b5f4bc33d2f480e3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733188565878
2024-12-03T01:16:08,341 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57504ad29584e935de6eb6630505b158#info#compaction#41 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:16:08,342 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/45d54135e8824781be521711e7e9456d is 1080, key is row0001/info:/1733188555734/Put/seqid=0
2024-12-03T01:16:08,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741842_1018 (size=44978)
2024-12-03T01:16:08,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741842_1018 (size=44978)
2024-12-03T01:16:08,354 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/45d54135e8824781be521711e7e9456d as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d
2024-12-03T01:16:08,362 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 57504ad29584e935de6eb6630505b158/info of 57504ad29584e935de6eb6630505b158 into 45d54135e8824781be521711e7e9456d(size=43.9 K), total size for store is 43.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:08,362 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158., storeName=57504ad29584e935de6eb6630505b158/info, priority=13, startTime=1733188568322; duration=0sec
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=43.9 K, sizeToCheck=16.0 K
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d because midkey is the same as first or last row
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:08,362 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57504ad29584e935de6eb6630505b158:info
2024-12-03T01:16:09,053 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:10,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:11,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:12,055 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:13,055 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:14,056 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:14,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:16:15,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:16,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:17,058 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:17,634 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:16:17,635 INFO  [RS-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:16:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:18,001 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:16:18,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6ed5e01f2384438bba3fd7aa45585704 is 1080, key is row0038/info:/1733188567890/Put/seqid=0
2024-12-03T01:16:18,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741843_1019 (size=29761)
2024-12-03T01:16:18,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741843_1019 (size=29761)
2024-12-03T01:16:18,012 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6ed5e01f2384438bba3fd7aa45585704
2024-12-03T01:16:18,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6ed5e01f2384438bba3fd7aa45585704 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704
2024-12-03T01:16:18,023 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704, entries=23, sequenceid=74, filesize=29.1 K
2024-12-03T01:16:18,024 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for 57504ad29584e935de6eb6630505b158 in 23ms, sequenceid=74, compaction requested=false
2024-12-03T01:16:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=73.0 K, sizeToCheck=16.0 K
2024-12-03T01:16:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d because midkey is the same as first or last row
2024-12-03T01:16:18,059 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:19,059 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,012 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:16:20,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/683f8d0875fa4feda738d00aadc16032 is 1080, key is row0061/info:/1733188578002/Put/seqid=0
2024-12-03T01:16:20,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741844_1020 (size=12509)
2024-12-03T01:16:20,026 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/683f8d0875fa4feda738d00aadc16032
2024-12-03T01:16:20,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741844_1020 (size=12509)
2024-12-03T01:16:20,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/683f8d0875fa4feda738d00aadc16032 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032
2024-12-03T01:16:20,042 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:16:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188590041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,042 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032, entries=7, sequenceid=84, filesize=12.2 K
2024-12-03T01:16:20,044 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 57504ad29584e935de6eb6630505b158 in 32ms, sequenceid=84, compaction requested=true
2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=85.2 K, sizeToCheck=16.0 K
2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d because midkey is the same as first or last row
2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 57504ad29584e935de6eb6630505b158:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:16:20,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,044 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:16:20,045 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 87248 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:16:20,045 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): 57504ad29584e935de6eb6630505b158/info is initiating minor compaction (all files)
2024-12-03T01:16:20,046 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 57504ad29584e935de6eb6630505b158/info in TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:20,046 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp, totalSize=85.2 K
2024-12-03T01:16:20,046 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 45d54135e8824781be521711e7e9456d, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733188555734
2024-12-03T01:16:20,047 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6ed5e01f2384438bba3fd7aa45585704, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733188567890
2024-12-03T01:16:20,047 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 683f8d0875fa4feda738d00aadc16032, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733188578002
2024-12-03T01:16:20,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:20,063 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 57504ad29584e935de6eb6630505b158#info#compaction#44 average throughput is 17.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:16:20,064 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/02755f039bf44c73bdfe2d15fc8e3e3d is 1080, key is row0001/info:/1733188555734/Put/seqid=0
2024-12-03T01:16:20,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741845_1021 (size=77532)
2024-12-03T01:16:20,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741845_1021 (size=77532)
2024-12-03T01:16:20,083 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/02755f039bf44c73bdfe2d15fc8e3e3d as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d
2024-12-03T01:16:20,088 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 57504ad29584e935de6eb6630505b158/info of 57504ad29584e935de6eb6630505b158 into 02755f039bf44c73bdfe2d15fc8e3e3d(size=75.7 K), total size for store is 75.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:16:20,089 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:20,089 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158., storeName=57504ad29584e935de6eb6630505b158/info, priority=13, startTime=1733188580044; duration=0sec
2024-12-03T01:16:20,089 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(109): Should split because info size=75.7 K, sizeToCheck=16.0 K
2024-12-03T01:16:20,089 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-03T01:16:20,090 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,090 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,090 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 57504ad29584e935de6eb6630505b158:info
2024-12-03T01:16:20,091 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44017 {}] assignment.AssignmentManager(1346): Split request from 1a7b1a4db65e,45383,1733188544276, parent={ENCODED => 57504ad29584e935de6eb6630505b158, NAME => 'TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-03T01:16:20,096 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44017 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,101 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44017 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=57504ad29584e935de6eb6630505b158, daughterA=c1d6cf602b426330b7916c2e5de4ab86, daughterB=d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,102 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=57504ad29584e935de6eb6630505b158, daughterA=c1d6cf602b426330b7916c2e5de4ab86, daughterB=d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,102 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=57504ad29584e935de6eb6630505b158, daughterA=c1d6cf602b426330b7916c2e5de4ab86, daughterB=d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,102 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=12, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=57504ad29584e935de6eb6630505b158, daughterA=c1d6cf602b426330b7916c2e5de4ab86, daughterB=d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,108 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, UNASSIGN}]
2024-12-03T01:16:20,109 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=13, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, UNASSIGN
2024-12-03T01:16:20,110 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=57504ad29584e935de6eb6630505b158, regionState=CLOSING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,112 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-03T01:16:20,112 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; CloseRegionProcedure 57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:16:20,266 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,268 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(124): Close 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,268 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(138): Unassign region: split region: true: evictCache: true
2024-12-03T01:16:20,269 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1681): Closing 57504ad29584e935de6eb6630505b158, disabling compactions & flushes
2024-12-03T01:16:20,269 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:20,269 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:20,269 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158. after waiting 0 ms
2024-12-03T01:16:20,269 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:20,269 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(2837): Flushing 57504ad29584e935de6eb6630505b158 1/1 column families, dataSize=23.12 KB heapSize=25 KB
2024-12-03T01:16:20,274 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6800cf46b8624e82984d3448f05eef9f is 1080, key is row0068/info:/1733188580012/Put/seqid=0
2024-12-03T01:16:20,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741846_1022 (size=28684)
2024-12-03T01:16:20,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741846_1022 (size=28684)
2024-12-03T01:16:20,280 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6800cf46b8624e82984d3448f05eef9f
2024-12-03T01:16:20,285 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/.tmp/info/6800cf46b8624e82984d3448f05eef9f as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6800cf46b8624e82984d3448f05eef9f
2024-12-03T01:16:20,289 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6800cf46b8624e82984d3448f05eef9f, entries=22, sequenceid=110, filesize=28.0 K
2024-12-03T01:16:20,290 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(3040): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=0 B/0 for 57504ad29584e935de6eb6630505b158 in 21ms, sequenceid=110, compaction requested=false
2024-12-03T01:16:20,291 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032] to archive
2024-12-03T01:16:20,292 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:16:20,294 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/7c5f3b52a0a248db967538de89525f98
2024-12-03T01:16:20,295 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/3b86c4e7e37f49b4a9caf6d8ac7431a3
2024-12-03T01:16:20,296 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/45d54135e8824781be521711e7e9456d
2024-12-03T01:16:20,297 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/9df512126e7143a4b5f4bc33d2f480e3
2024-12-03T01:16:20,298 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6ed5e01f2384438bba3fd7aa45585704
2024-12-03T01:16:20,300 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/683f8d0875fa4feda738d00aadc16032
2024-12-03T01:16:20,305 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=1
2024-12-03T01:16:20,305 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.
2024-12-03T01:16:20,305 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] regionserver.HRegion(1635): Region close journal for 57504ad29584e935de6eb6630505b158:

2024-12-03T01:16:20,307 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION, pid=14}] handler.UnassignRegionHandler(170): Closed 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,307 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=13 updating hbase:meta row=57504ad29584e935de6eb6630505b158, regionState=CLOSED
2024-12-03T01:16:20,310 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13
2024-12-03T01:16:20,310 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; CloseRegionProcedure 57504ad29584e935de6eb6630505b158, server=1a7b1a4db65e,45383,1733188544276 in 197 msec
2024-12-03T01:16:20,312 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12
2024-12-03T01:16:20,312 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=57504ad29584e935de6eb6630505b158, UNASSIGN in 202 msec
2024-12-03T01:16:20,332 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:16:20,333 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=12 splitting 2 storefiles, region=57504ad29584e935de6eb6630505b158, threads=2
2024-12-03T01:16:20,334 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d for region: 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,334 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=12 splitting started for store file: hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6800cf46b8624e82984d3448f05eef9f for region: 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,343 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6800cf46b8624e82984d3448f05eef9f, top=true
2024-12-03T01:16:20,347 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f for child: d47329c78b19d07e1df6eca553401e59, parent: 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,347 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/6800cf46b8624e82984d3448f05eef9f for region: 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741847_1023 (size=27)
2024-12-03T01:16:20,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741847_1023 (size=27)
2024-12-03T01:16:20,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741848_1024 (size=27)
2024-12-03T01:16:20,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741848_1024 (size=27)
2024-12-03T01:16:20,359 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=12 splitting complete for store file: hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d for region: 57504ad29584e935de6eb6630505b158
2024-12-03T01:16:20,359 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=12 split storefiles for region 57504ad29584e935de6eb6630505b158 Daughter A: [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158] storefiles, Daughter B: [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f] storefiles.
2024-12-03T01:16:20,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741849_1025 (size=71)
2024-12-03T01:16:20,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741849_1025 (size=71)
2024-12-03T01:16:20,368 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:16:20,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741850_1026 (size=71)
2024-12-03T01:16:20,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741850_1026 (size=71)
2024-12-03T01:16:20,381 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:16:20,393 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=-1
2024-12-03T01:16:20,396 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/recovered.edits/113.seqid, newMaxSeqId=113, maxSeqId=-1
2024-12-03T01:16:20,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733188580398"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733188580398"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733188580398"}]},"ts":"1733188580398"}
2024-12-03T01:16:20,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733188580398"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188580398"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733188580398"}]},"ts":"1733188580398"}
2024-12-03T01:16:20,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733188580398"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188580398"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733188580398"}]},"ts":"1733188580398"}
2024-12-03T01:16:20,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45383 {}] regionserver.HRegion(8581): Flush requested on 1588230740
2024-12-03T01:16:20,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-03T01:16:20,432 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=4.75 KB heapSize=8.29 KB
2024-12-03T01:16:20,436 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c1d6cf602b426330b7916c2e5de4ab86, ASSIGN}, {pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d47329c78b19d07e1df6eca553401e59, ASSIGN}]
2024-12-03T01:16:20,437 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d47329c78b19d07e1df6eca553401e59, ASSIGN
2024-12-03T01:16:20,437 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c1d6cf602b426330b7916c2e5de4ab86, ASSIGN
2024-12-03T01:16:20,438 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=15, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c1d6cf602b426330b7916c2e5de4ab86, ASSIGN; state=SPLITTING_NEW, location=1a7b1a4db65e,45383,1733188544276; forceNewPlan=false, retain=false
2024-12-03T01:16:20,438 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=16, ppid=12, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d47329c78b19d07e1df6eca553401e59, ASSIGN; state=SPLITTING_NEW, location=1a7b1a4db65e,45383,1733188544276; forceNewPlan=false, retain=false
2024-12-03T01:16:20,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/c021cd2b18714166820e8dc2e7d8e0bd is 193, key is TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59./info:regioninfo/1733188580398/Put/seqid=0
2024-12-03T01:16:20,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741851_1027 (size=9423)
2024-12-03T01:16:20,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741851_1027 (size=9423)
2024-12-03T01:16:20,454 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.54 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/c021cd2b18714166820e8dc2e7d8e0bd
2024-12-03T01:16:20,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/table/a9fc1584bea44eceabc44bb992944443 is 65, key is TestLogRolling-testLogRolling/table:state/1733188546152/Put/seqid=0
2024-12-03T01:16:20,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741852_1028 (size=5412)
2024-12-03T01:16:20,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741852_1028 (size=5412)
2024-12-03T01:16:20,479 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=216 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/table/a9fc1584bea44eceabc44bb992944443
2024-12-03T01:16:20,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/c021cd2b18714166820e8dc2e7d8e0bd as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/info/c021cd2b18714166820e8dc2e7d8e0bd
2024-12-03T01:16:20,491 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/info/c021cd2b18714166820e8dc2e7d8e0bd, entries=29, sequenceid=17, filesize=9.2 K
2024-12-03T01:16:20,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/table/a9fc1584bea44eceabc44bb992944443 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/table/a9fc1584bea44eceabc44bb992944443
2024-12-03T01:16:20,497 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/table/a9fc1584bea44eceabc44bb992944443, entries=4, sequenceid=17, filesize=5.3 K
2024-12-03T01:16:20,498 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~4.75 KB/4869, heapSize ~8.01 KB/8200, currentSize=0 B/0 for 1588230740 in 65ms, sequenceid=17, compaction requested=false
2024-12-03T01:16:20,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-03T01:16:20,588 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=c1d6cf602b426330b7916c2e5de4ab86, regionState=OPENING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,588 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=d47329c78b19d07e1df6eca553401e59, regionState=OPENING, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,590 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=15, state=RUNNABLE; OpenRegionProcedure c1d6cf602b426330b7916c2e5de4ab86, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:16:20,591 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=16, state=RUNNABLE; OpenRegionProcedure d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276}]
2024-12-03T01:16:20,743 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,747 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:16:20,747 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7285): Opening region: {ENCODED => c1d6cf602b426330b7916c2e5de4ab86, NAME => 'TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-03T01:16:20,747 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,747 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:16:20,747 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7327): checking encryption for c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,747 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7330): checking classloading for c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,749 INFO  [StoreOpener-c1d6cf602b426330b7916c2e5de4ab86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c1d6cf602b426330b7916c2e5de4ab86 
2024-12-03T01:16:20,750 INFO  [StoreOpener-c1d6cf602b426330b7916c2e5de4ab86-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1d6cf602b426330b7916c2e5de4ab86 columnFamilyName info
2024-12-03T01:16:20,750 DEBUG [StoreOpener-c1d6cf602b426330b7916c2e5de4ab86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:16:20,764 DEBUG [StoreOpener-c1d6cf602b426330b7916c2e5de4ab86-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-bottom
2024-12-03T01:16:20,765 INFO  [StoreOpener-c1d6cf602b426330b7916c2e5de4ab86-1 {}] regionserver.HStore(327): Store=c1d6cf602b426330b7916c2e5de4ab86/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:16:20,766 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,767 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,769 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1085): writing seq id for c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:16:20,771 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1102): Opened c1d6cf602b426330b7916c2e5de4ab86; next sequenceid=114; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809221, jitterRate=0.02897825837135315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:16:20,771 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1001): Region open journal for c1d6cf602b426330b7916c2e5de4ab86:

2024-12-03T01:16:20,772 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86., pid=17, masterSystemTime=1733188580743
2024-12-03T01:16:20,773 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(403): Add compact mark for store c1d6cf602b426330b7916c2e5de4ab86:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:16:20,773 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,773 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-03T01:16:20,773 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:16:20,774 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): c1d6cf602b426330b7916c2e5de4ab86/info is initiating minor compaction (all files)
2024-12-03T01:16:20,774 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of c1d6cf602b426330b7916c2e5de4ab86/info in TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:16:20,774 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-bottom] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/.tmp, totalSize=75.7 K
2024-12-03T01:16:20,774 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733188555734
2024-12-03T01:16:20,774 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:16:20,775 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:16:20,775 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(135): Open TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:20,775 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7285): Opening region: {ENCODED => d47329c78b19d07e1df6eca553401e59, NAME => 'TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-03T01:16:20,775 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=15 updating hbase:meta row=c1d6cf602b426330b7916c2e5de4ab86, regionState=OPEN, openSeqNum=114, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,775 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,775 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(894): Instantiated TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:16:20,775 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7327): checking encryption for d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,775 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7330): checking classloading for d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,777 INFO  [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d47329c78b19d07e1df6eca553401e59 
2024-12-03T01:16:20,778 INFO  [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d47329c78b19d07e1df6eca553401e59 columnFamilyName info
2024-12-03T01:16:20,778 DEBUG [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:16:20,780 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=15
2024-12-03T01:16:20,780 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=15, state=SUCCESS; OpenRegionProcedure c1d6cf602b426330b7916c2e5de4ab86, server=1a7b1a4db65e,45383,1733188544276 in 188 msec
2024-12-03T01:16:20,782 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c1d6cf602b426330b7916c2e5de4ab86, ASSIGN in 344 msec
2024-12-03T01:16:20,788 DEBUG [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-top
2024-12-03T01:16:20,793 DEBUG [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] regionserver.StoreEngine(277): loaded hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f
2024-12-03T01:16:20,793 INFO  [StoreOpener-d47329c78b19d07e1df6eca553401e59-1 {}] regionserver.HStore(327): Store=d47329c78b19d07e1df6eca553401e59/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:16:20,794 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,795 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,797 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1d6cf602b426330b7916c2e5de4ab86#info#compaction#48 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:16:20,798 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/.tmp/info/f431544138054ed5b76c917fbeb2d173 is 1080, key is row0001/info:/1733188555734/Put/seqid=0
2024-12-03T01:16:20,798 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1085): writing seq id for d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:20,799 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1102): Opened d47329c78b19d07e1df6eca553401e59; next sequenceid=114; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=764059, jitterRate=-0.02844955027103424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:16:20,799 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1001): Region open journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:16:20,800 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2601): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., pid=18, masterSystemTime=1733188580743
2024-12-03T01:16:20,801 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 2
2024-12-03T01:16:20,801 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,801 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking
2024-12-03T01:16:20,802 DEBUG [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2628): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:20,802 INFO  [RS_OPEN_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(164): Opened TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:20,803 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=16 updating hbase:meta row=d47329c78b19d07e1df6eca553401e59, regionState=OPEN, openSeqNum=114, regionLocation=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:20,804 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HStore(1526): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:20,804 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:16:20,804 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:20,805 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-top, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=103.7 K
2024-12-03T01:16:20,805 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] compactions.Compactor(224): Compacting 02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158, keycount=33, bloomtype=ROW, size=75.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733188555734
2024-12-03T01:16:20,806 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] compactions.Compactor(224): Compacting TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733188580012
2024-12-03T01:16:20,807 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=16
2024-12-03T01:16:20,807 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=16, state=SUCCESS; OpenRegionProcedure d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276 in 214 msec
2024-12-03T01:16:20,809 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=12
2024-12-03T01:16:20,809 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=12, state=SUCCESS; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d47329c78b19d07e1df6eca553401e59, ASSIGN in 371 msec
2024-12-03T01:16:20,811 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=57504ad29584e935de6eb6630505b158, daughterA=c1d6cf602b426330b7916c2e5de4ab86, daughterB=d47329c78b19d07e1df6eca553401e59 in 713 msec
2024-12-03T01:16:20,814 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741853_1029 (size=70862)
2024-12-03T01:16:20,815 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741853_1029 (size=70862)
2024-12-03T01:16:20,824 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/.tmp/info/f431544138054ed5b76c917fbeb2d173 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/f431544138054ed5b76c917fbeb2d173
2024-12-03T01:16:20,835 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-03T01:16:20,836 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 1 (all) file(s) in c1d6cf602b426330b7916c2e5de4ab86/info of c1d6cf602b426330b7916c2e5de4ab86 into f431544138054ed5b76c917fbeb2d173(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:16:20,836 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for c1d6cf602b426330b7916c2e5de4ab86:

2024-12-03T01:16:20,836 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86., storeName=c1d6cf602b426330b7916c2e5de4ab86/info, priority=15, startTime=1733188580773; duration=0sec
2024-12-03T01:16:20,836 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,836 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1d6cf602b426330b7916c2e5de4ab86:info
2024-12-03T01:16:20,841 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#49 average throughput is 28.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:16:20,841 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/ebb61fc1058a40bcbad66395a5086638 is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:16:20,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741854_1030 (size=35344)
2024-12-03T01:16:20,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741854_1030 (size=35344)
2024-12-03T01:16:20,853 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/ebb61fc1058a40bcbad66395a5086638 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/ebb61fc1058a40bcbad66395a5086638
2024-12-03T01:16:20,859 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 2 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into ebb61fc1058a40bcbad66395a5086638(size=34.5 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:16:20,859 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:16:20,859 INFO  [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=14, startTime=1733188580801; duration=0sec
2024-12-03T01:16:20,859 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:20,859 DEBUG [RS:0;1a7b1a4db65e:45383-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:16:21,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:22,061 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:23,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:24,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:25,063 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,306 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,323 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,325 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,326 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,326 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,327 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,835 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-03T01:16:25,836 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,836 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,837 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,855 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-03T01:16:25,904 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-03T01:16:25,904 INFO  [master/1a7b1a4db65e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-03T01:16:26,063 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:27,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:28,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:29,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188600055, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733188545722.57504ad29584e935de6eb6630505b158. is not online on 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:30,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:30,197 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835
2024-12-03T01:16:31,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:32,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:33,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:34,068 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:35,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:36,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:37,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:38,071 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:39,072 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:39,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 23930
2024-12-03T01:16:40,072 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:41,073 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:42,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:43,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:44,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:44,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:16:45,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:46,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:46,449 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 97f43709f8e893c79b431e7315d66f1f changed from -1.0 to 0.0, refreshing cache
2024-12-03T01:16:47,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:48,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:49,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:50,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:50,097 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:16:50,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/241b288218034247ad067277c05d7eb1 is 1080, key is row0090/info:/1733188610089/Put/seqid=0
2024-12-03T01:16:50,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741855_1031 (size=12509)
2024-12-03T01:16:50,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741855_1031 (size=12509)
2024-12-03T01:16:50,111 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/241b288218034247ad067277c05d7eb1
2024-12-03T01:16:50,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/241b288218034247ad067277c05d7eb1 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1
2024-12-03T01:16:50,122 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1, entries=7, sequenceid=124, filesize=12.2 K
2024-12-03T01:16:50,123 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d47329c78b19d07e1df6eca553401e59 in 25ms, sequenceid=124, compaction requested=false
2024-12-03T01:16:50,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:16:51,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:52,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:16:52,106 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:16:52,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/7d4963c0d4f240eeb0df36b48fe6a2a8 is 1080, key is row0097/info:/1733188612098/Put/seqid=0
2024-12-03T01:16:52,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741856_1032 (size=12515)
2024-12-03T01:16:52,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741856_1032 (size=12515)
2024-12-03T01:16:52,133 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:16:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188622133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:16:52,518 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/7d4963c0d4f240eeb0df36b48fe6a2a8
2024-12-03T01:16:52,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/7d4963c0d4f240eeb0df36b48fe6a2a8 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8
2024-12-03T01:16:52,530 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8, entries=7, sequenceid=134, filesize=12.2 K
2024-12-03T01:16:52,531 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d47329c78b19d07e1df6eca553401e59 in 425ms, sequenceid=134, compaction requested=true
2024-12-03T01:16:52,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:52,532 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:16:52,533 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 60368 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:16:52,533 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:16:52,533 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:16:52,533 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/ebb61fc1058a40bcbad66395a5086638, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=59.0 K
2024-12-03T01:16:52,534 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting ebb61fc1058a40bcbad66395a5086638, keycount=28, bloomtype=ROW, size=34.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733188578004
2024-12-03T01:16:52,534 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 241b288218034247ad067277c05d7eb1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733188610089
2024-12-03T01:16:52,534 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7d4963c0d4f240eeb0df36b48fe6a2a8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733188612098
2024-12-03T01:16:52,547 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#52 average throughput is 43.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:16:52,548 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/b09379dafd6d4394930e188f8fe548a3 is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:16:52,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741857_1033 (size=50534)
2024-12-03T01:16:52,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741857_1033 (size=50534)
2024-12-03T01:16:52,560 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/b09379dafd6d4394930e188f8fe548a3 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/b09379dafd6d4394930e188f8fe548a3
2024-12-03T01:16:52,566 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into b09379dafd6d4394930e188f8fe548a3(size=49.3 K), total size for store is 49.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:16:52,566 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:16:52,566 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188612532; duration=0sec
2024-12-03T01:16:52,566 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:16:52,566 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:16:53,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:54,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:55,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:56,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:57,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:57,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=3 on file=hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta after 196137ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor199.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-03T01:16:58,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:16:59,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:00,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:01,086 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:02,086 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:02,228 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:17:02,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/e9d5e5fad1c14570bee169d5a682a977 is 1080, key is row0104/info:/1733188612106/Put/seqid=0
2024-12-03T01:17:02,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741858_1034 (size=29784)
2024-12-03T01:17:02,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741858_1034 (size=29784)
2024-12-03T01:17:02,256 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/e9d5e5fad1c14570bee169d5a682a977
2024-12-03T01:17:02,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/e9d5e5fad1c14570bee169d5a682a977 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977
2024-12-03T01:17:02,292 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977, entries=23, sequenceid=161, filesize=29.1 K
2024-12-03T01:17:02,305 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for d47329c78b19d07e1df6eca553401e59 in 77ms, sequenceid=161, compaction requested=false
2024-12-03T01:17:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:03,087 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:04,087 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:04,237 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:17:04,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/5cfe7b5c347c4483a9f3f04e6890a739 is 1080, key is row0127/info:/1733188622228/Put/seqid=0
2024-12-03T01:17:04,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741859_1035 (size=12516)
2024-12-03T01:17:04,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741859_1035 (size=12516)
2024-12-03T01:17:04,247 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/5cfe7b5c347c4483a9f3f04e6890a739
2024-12-03T01:17:04,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/5cfe7b5c347c4483a9f3f04e6890a739 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739
2024-12-03T01:17:04,259 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739, entries=7, sequenceid=171, filesize=12.2 K
2024-12-03T01:17:04,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d47329c78b19d07e1df6eca553401e59 in 22ms, sequenceid=171, compaction requested=true
2024-12-03T01:17:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:17:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:04,260 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:04,260 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:17:04,261 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 92834 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:17:04,261 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:17:04,261 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:04,261 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/b09379dafd6d4394930e188f8fe548a3, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=90.7 K
2024-12-03T01:17:04,262 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting b09379dafd6d4394930e188f8fe548a3, keycount=42, bloomtype=ROW, size=49.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733188578004
2024-12-03T01:17:04,262 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting e9d5e5fad1c14570bee169d5a682a977, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733188612106
2024-12-03T01:17:04,262 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5cfe7b5c347c4483a9f3f04e6890a739, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733188622228
2024-12-03T01:17:04,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/d73490f1e28040be86b7a597c3b3a8e4 is 1080, key is row0134/info:/1733188624237/Put/seqid=0
2024-12-03T01:17:04,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741860_1036 (size=29784)
2024-12-03T01:17:04,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741860_1036 (size=29784)
2024-12-03T01:17:04,272 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/d73490f1e28040be86b7a597c3b3a8e4
2024-12-03T01:17:04,275 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#56 average throughput is 73.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:17:04,276 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/6532444ec46442d8a95ed483c22a032b is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:17:04,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/d73490f1e28040be86b7a597c3b3a8e4 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4
2024-12-03T01:17:04,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741861_1037 (size=83117)
2024-12-03T01:17:04,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741861_1037 (size=83117)
2024-12-03T01:17:04,283 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4, entries=23, sequenceid=197, filesize=29.1 K
2024-12-03T01:17:04,284 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=4.20 KB/4304 for d47329c78b19d07e1df6eca553401e59 in 24ms, sequenceid=197, compaction requested=false
2024-12-03T01:17:04,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:04,286 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/6532444ec46442d8a95ed483c22a032b as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6532444ec46442d8a95ed483c22a032b
2024-12-03T01:17:04,291 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into 6532444ec46442d8a95ed483c22a032b(size=81.2 K), total size for store is 110.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:17:04,291 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:04,291 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188624260; duration=0sec
2024-12-03T01:17:04,291 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:04,291 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:17:05,088 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:05,747 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region c1d6cf602b426330b7916c2e5de4ab86, had cached 0 bytes from a total of 70862
2024-12-03T01:17:05,775 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region d47329c78b19d07e1df6eca553401e59, had cached 0 bytes from a total of 112901
2024-12-03T01:17:06,088 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:06,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:06,269 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:17:06,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a891485e1de44653bfca22da080f7e91 is 1080, key is row0157/info:/1733188624261/Put/seqid=0
2024-12-03T01:17:06,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741862_1038 (size=12516)
2024-12-03T01:17:06,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741862_1038 (size=12516)
2024-12-03T01:17:06,290 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a891485e1de44653bfca22da080f7e91
2024-12-03T01:17:06,291 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:17:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188636291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:06,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a891485e1de44653bfca22da080f7e91 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91
2024-12-03T01:17:06,301 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91, entries=7, sequenceid=208, filesize=12.2 K
2024-12-03T01:17:06,302 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d47329c78b19d07e1df6eca553401e59 in 33ms, sequenceid=208, compaction requested=true
2024-12-03T01:17:06,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:06,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:17:06,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:06,302 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:17:06,303 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 125417 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:17:06,303 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:17:06,303 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:06,303 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6532444ec46442d8a95ed483c22a032b, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=122.5 K
2024-12-03T01:17:06,304 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6532444ec46442d8a95ed483c22a032b, keycount=72, bloomtype=ROW, size=81.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733188578004
2024-12-03T01:17:06,304 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting d73490f1e28040be86b7a597c3b3a8e4, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733188624237
2024-12-03T01:17:06,305 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting a891485e1de44653bfca22da080f7e91, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733188624261
2024-12-03T01:17:06,317 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#58 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:17:06,317 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/afb728e9b8474ca6a43ac5be9b909ff8 is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:17:06,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741863_1039 (size=115567)
2024-12-03T01:17:06,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741863_1039 (size=115567)
2024-12-03T01:17:06,328 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/afb728e9b8474ca6a43ac5be9b909ff8 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/afb728e9b8474ca6a43ac5be9b909ff8
2024-12-03T01:17:06,335 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into afb728e9b8474ca6a43ac5be9b909ff8(size=112.9 K), total size for store is 112.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:17:06,335 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:06,335 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188626302; duration=0sec
2024-12-03T01:17:06,335 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:06,335 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:17:07,089 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:08,090 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:09,090 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:10,091 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:11,091 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:12,092 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:13,092 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:14,093 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:14,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:17:15,094 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:15,197 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 14835
2024-12-03T01:17:16,094 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:16,374 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:17:16,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/6ccf016bd01a428c804fbd3c896ad923 is 1080, key is row0164/info:/1733188626270/Put/seqid=0
2024-12-03T01:17:16,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741864_1040 (size=29784)
2024-12-03T01:17:16,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741864_1040 (size=29784)
2024-12-03T01:17:16,388 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/6ccf016bd01a428c804fbd3c896ad923
2024-12-03T01:17:16,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/6ccf016bd01a428c804fbd3c896ad923 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923
2024-12-03T01:17:16,399 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923, entries=23, sequenceid=235, filesize=29.1 K
2024-12-03T01:17:16,400 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for d47329c78b19d07e1df6eca553401e59 in 26ms, sequenceid=235, compaction requested=false
2024-12-03T01:17:16,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:17,095 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:18,095 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:18,384 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:17:18,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a17f17453409430fa8abb960a0e7a6db is 1080, key is row0187/info:/1733188636375/Put/seqid=0
2024-12-03T01:17:18,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741865_1041 (size=12516)
2024-12-03T01:17:18,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741865_1041 (size=12516)
2024-12-03T01:17:18,403 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a17f17453409430fa8abb960a0e7a6db
2024-12-03T01:17:18,408 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:17:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188648407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:18,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a17f17453409430fa8abb960a0e7a6db as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db
2024-12-03T01:17:18,413 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db, entries=7, sequenceid=245, filesize=12.2 K
2024-12-03T01:17:18,414 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d47329c78b19d07e1df6eca553401e59 in 31ms, sequenceid=245, compaction requested=true
2024-12-03T01:17:18,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:18,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:17:18,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:18,414 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:17:18,415 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 157867 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:17:18,415 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:17:18,415 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:18,415 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/afb728e9b8474ca6a43ac5be9b909ff8, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=154.2 K
2024-12-03T01:17:18,416 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting afb728e9b8474ca6a43ac5be9b909ff8, keycount=102, bloomtype=ROW, size=112.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733188578004
2024-12-03T01:17:18,416 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6ccf016bd01a428c804fbd3c896ad923, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733188626270
2024-12-03T01:17:18,416 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting a17f17453409430fa8abb960a0e7a6db, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733188636375
2024-12-03T01:17:18,428 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#61 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:17:18,428 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/01a3cb4f323148c088039c2b744ccbbe is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:17:18,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741866_1042 (size=148214)
2024-12-03T01:17:18,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741866_1042 (size=148214)
2024-12-03T01:17:18,444 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/01a3cb4f323148c088039c2b744ccbbe as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/01a3cb4f323148c088039c2b744ccbbe
2024-12-03T01:17:18,450 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into 01a3cb4f323148c088039c2b744ccbbe(size=144.7 K), total size for store is 144.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:17:18,450 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:18,450 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188638414; duration=0sec
2024-12-03T01:17:18,450 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:18,450 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:17:19,096 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:20,097 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:21,097 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:22,098 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:23,098 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:24,099 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:24,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 4b99be8732eddce4ec2a4c71b945ea2c, had cached 0 bytes from a total of 23930
2024-12-03T01:17:25,099 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:26,100 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:27,101 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:28,101 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:28,451 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:17:28,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/56a328c50dfc4f72b5b5049bd74eb474 is 1080, key is row0194/info:/1733188638384/Put/seqid=0
2024-12-03T01:17:28,457 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:17:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188658457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:28,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741867_1043 (size=29798)
2024-12-03T01:17:28,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741867_1043 (size=29798)
2024-12-03T01:17:28,461 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/56a328c50dfc4f72b5b5049bd74eb474
2024-12-03T01:17:28,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/56a328c50dfc4f72b5b5049bd74eb474 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474
2024-12-03T01:17:28,471 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474, entries=23, sequenceid=272, filesize=29.1 K
2024-12-03T01:17:28,471 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=6.30 KB/6456 for d47329c78b19d07e1df6eca553401e59 in 20ms, sequenceid=272, compaction requested=false
2024-12-03T01:17:28,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:29,102 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:30,102 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:31,103 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:32,103 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:33,104 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:34,105 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:35,105 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:36,106 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:37,106 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:38,107 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:38,512 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:17:38,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/3976489648ad498d9416eb014205ea44 is 1080, key is row0217/info:/1733188648451/Put/seqid=0
2024-12-03T01:17:38,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741868_1044 (size=12523)
2024-12-03T01:17:38,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741868_1044 (size=12523)
2024-12-03T01:17:38,522 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/3976489648ad498d9416eb014205ea44
2024-12-03T01:17:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/3976489648ad498d9416eb014205ea44 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44
2024-12-03T01:17:38,533 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44, entries=7, sequenceid=282, filesize=12.2 K
2024-12-03T01:17:38,534 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=1.05 KB/1076 for d47329c78b19d07e1df6eca553401e59 in 21ms, sequenceid=282, compaction requested=true
2024-12-03T01:17:38,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:38,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:17:38,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:38,534 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:17:38,535 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 190535 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:17:38,535 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:17:38,535 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:38,535 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/01a3cb4f323148c088039c2b744ccbbe, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=186.1 K
2024-12-03T01:17:38,535 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 01a3cb4f323148c088039c2b744ccbbe, keycount=132, bloomtype=ROW, size=144.7 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733188578004
2024-12-03T01:17:38,535 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 56a328c50dfc4f72b5b5049bd74eb474, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733188638384
2024-12-03T01:17:38,536 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3976489648ad498d9416eb014205ea44, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733188648451
2024-12-03T01:17:38,547 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#64 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:17:38,547 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/fc0eac3466c440248de9fc4f1fb4882c is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:17:38,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741869_1045 (size=180685)
2024-12-03T01:17:38,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741869_1045 (size=180685)
2024-12-03T01:17:38,558 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/fc0eac3466c440248de9fc4f1fb4882c as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/fc0eac3466c440248de9fc4f1fb4882c
2024-12-03T01:17:38,563 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into fc0eac3466c440248de9fc4f1fb4882c(size=176.5 K), total size for store is 176.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:17:38,563 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:38,563 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188658534; duration=0sec
2024-12-03T01:17:38,563 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:38,563 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:17:39,108 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:40,108 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:40,521 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-03T01:17:40,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/2d8b12285f8e4cb5a2b595043ac37e29 is 1080, key is row0224/info:/1733188658512/Put/seqid=0
2024-12-03T01:17:40,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741870_1046 (size=12523)
2024-12-03T01:17:40,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741870_1046 (size=12523)
2024-12-03T01:17:40,535 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/2d8b12285f8e4cb5a2b595043ac37e29
2024-12-03T01:17:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/2d8b12285f8e4cb5a2b595043ac37e29 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29
2024-12-03T01:17:40,545 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT]
2024-12-03T01:17:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:51986 deadline: 1733188670545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d47329c78b19d07e1df6eca553401e59, server=1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:40,548 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29, entries=7, sequenceid=293, filesize=12.2 K
2024-12-03T01:17:40,549 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d47329c78b19d07e1df6eca553401e59 in 28ms, sequenceid=293, compaction requested=false
2024-12-03T01:17:40,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:41,109 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:42,109 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:43,110 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:44,097 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-03T01:17:44,111 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:44,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-top, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/ebb61fc1058a40bcbad66395a5086638, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/b09379dafd6d4394930e188f8fe548a3, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6532444ec46442d8a95ed483c22a032b, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/afb728e9b8474ca6a43ac5be9b909ff8, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/01a3cb4f323148c088039c2b744ccbbe, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44] to archive
2024-12-03T01:17:44,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:17:44,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158
2024-12-03T01:17:44,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/ebb61fc1058a40bcbad66395a5086638 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/ebb61fc1058a40bcbad66395a5086638
2024-12-03T01:17:44,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/TestLogRolling-testLogRolling=57504ad29584e935de6eb6630505b158-6800cf46b8624e82984d3448f05eef9f
2024-12-03T01:17:44,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/241b288218034247ad067277c05d7eb1
2024-12-03T01:17:44,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/b09379dafd6d4394930e188f8fe548a3 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/b09379dafd6d4394930e188f8fe548a3
2024-12-03T01:17:44,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/7d4963c0d4f240eeb0df36b48fe6a2a8
2024-12-03T01:17:44,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/e9d5e5fad1c14570bee169d5a682a977
2024-12-03T01:17:44,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6532444ec46442d8a95ed483c22a032b to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6532444ec46442d8a95ed483c22a032b
2024-12-03T01:17:44,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/5cfe7b5c347c4483a9f3f04e6890a739
2024-12-03T01:17:44,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/d73490f1e28040be86b7a597c3b3a8e4
2024-12-03T01:17:44,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/afb728e9b8474ca6a43ac5be9b909ff8 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/afb728e9b8474ca6a43ac5be9b909ff8
2024-12-03T01:17:44,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a891485e1de44653bfca22da080f7e91
2024-12-03T01:17:44,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/6ccf016bd01a428c804fbd3c896ad923
2024-12-03T01:17:44,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/01a3cb4f323148c088039c2b744ccbbe to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/01a3cb4f323148c088039c2b744ccbbe
2024-12-03T01:17:44,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a17f17453409430fa8abb960a0e7a6db
2024-12-03T01:17:44,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/56a328c50dfc4f72b5b5049bd74eb474
2024-12-03T01:17:44,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/3976489648ad498d9416eb014205ea44
2024-12-03T01:17:44,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158->hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/57504ad29584e935de6eb6630505b158/info/02755f039bf44c73bdfe2d15fc8e3e3d-bottom] to archive
2024-12-03T01:17:44,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:17:44,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/info/02755f039bf44c73bdfe2d15fc8e3e3d.57504ad29584e935de6eb6630505b158
2024-12-03T01:17:45,111 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:46,112 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:47,112 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:48,113 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:49,113 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:50,114 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.HRegion(8581): Flush requested on d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:50,551 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-03T01:17:50,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a8ce520720024d71b42ddd92d2ba47bb is 1080, key is row0231/info:/1733188660521/Put/seqid=0
2024-12-03T01:17:50,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741871_1047 (size=29807)
2024-12-03T01:17:50,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741871_1047 (size=29807)
2024-12-03T01:17:50,560 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a8ce520720024d71b42ddd92d2ba47bb
2024-12-03T01:17:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/a8ce520720024d71b42ddd92d2ba47bb as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb
2024-12-03T01:17:50,571 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb, entries=23, sequenceid=319, filesize=29.1 K
2024-12-03T01:17:50,572 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=3.15 KB/3228 for d47329c78b19d07e1df6eca553401e59 in 21ms, sequenceid=319, compaction requested=true
2024-12-03T01:17:50,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:50,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d47329c78b19d07e1df6eca553401e59:info, priority=-2147483648, current under compaction store size is 1
2024-12-03T01:17:50,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:50,572 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-03T01:17:50,573 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 223015 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-03T01:17:50,573 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1540): d47329c78b19d07e1df6eca553401e59/info is initiating minor compaction (all files)
2024-12-03T01:17:50,573 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d47329c78b19d07e1df6eca553401e59/info in TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:50,573 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/fc0eac3466c440248de9fc4f1fb4882c, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb] into tmpdir=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp, totalSize=217.8 K
2024-12-03T01:17:50,574 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting fc0eac3466c440248de9fc4f1fb4882c, keycount=162, bloomtype=ROW, size=176.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733188578004
2024-12-03T01:17:50,574 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2d8b12285f8e4cb5a2b595043ac37e29, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1733188658512
2024-12-03T01:17:50,574 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] compactions.Compactor(224): Compacting a8ce520720024d71b42ddd92d2ba47bb, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733188660521
2024-12-03T01:17:50,585 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d47329c78b19d07e1df6eca553401e59#info#compaction#67 average throughput is 98.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-03T01:17:50,586 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/4788a437e20a45d3813a203b153b2f0e is 1080, key is row0062/info:/1733188578004/Put/seqid=0
2024-12-03T01:17:50,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741872_1048 (size=212559)
2024-12-03T01:17:50,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741872_1048 (size=212559)
2024-12-03T01:17:50,594 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/4788a437e20a45d3813a203b153b2f0e as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/4788a437e20a45d3813a203b153b2f0e
2024-12-03T01:17:50,599 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d47329c78b19d07e1df6eca553401e59/info of d47329c78b19d07e1df6eca553401e59 into 4788a437e20a45d3813a203b153b2f0e(size=207.6 K), total size for store is 207.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-03T01:17:50,599 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:50,599 INFO  [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., storeName=d47329c78b19d07e1df6eca553401e59/info, priority=13, startTime=1733188670572; duration=0sec
2024-12-03T01:17:50,599 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-03T01:17:50,599 DEBUG [RS:0;1a7b1a4db65e:45383-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d47329c78b19d07e1df6eca553401e59:info
2024-12-03T01:17:50,748 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region c1d6cf602b426330b7916c2e5de4ab86, had cached 0 bytes from a total of 70862
2024-12-03T01:17:50,776 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region d47329c78b19d07e1df6eca553401e59, had cached 0 bytes from a total of 212559
2024-12-03T01:17:51,115 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:52,115 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:52,554 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-03T01:17:52,555 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C45383%2C1733188544276.1733188672554
2024-12-03T01:17:52,562 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188544803 with entries=309, filesize=306.80 KB; new WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188672554
2024-12-03T01:17:52,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33535:33535),(127.0.0.1/127.0.0.1:45751:45751)]
2024-12-03T01:17:52,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188544803 is not closed yet, will try archiving it next time
2024-12-03T01:17:52,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741833_1009 (size=314174)
2024-12-03T01:17:52,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741833_1009 (size=314174)
2024-12-03T01:17:52,569 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 97f43709f8e893c79b431e7315d66f1f 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:17:52,592 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/.tmp/info/a42a8a3b1433443e86aafbfeb1dc2f2c is 45, key is default/info:d/1733188545648/Put/seqid=0
2024-12-03T01:17:52,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741874_1050 (size=5037)
2024-12-03T01:17:52,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741874_1050 (size=5037)
2024-12-03T01:17:52,598 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/.tmp/info/a42a8a3b1433443e86aafbfeb1dc2f2c
2024-12-03T01:17:52,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/.tmp/info/a42a8a3b1433443e86aafbfeb1dc2f2c as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/info/a42a8a3b1433443e86aafbfeb1dc2f2c
2024-12-03T01:17:52,609 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/info/a42a8a3b1433443e86aafbfeb1dc2f2c, entries=2, sequenceid=6, filesize=4.9 K
2024-12-03T01:17:52,611 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 97f43709f8e893c79b431e7315d66f1f in 42ms, sequenceid=6, compaction requested=false
2024-12-03T01:17:52,611 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 97f43709f8e893c79b431e7315d66f1f:

2024-12-03T01:17:52,611 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing d47329c78b19d07e1df6eca553401e59 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-03T01:17:52,616 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/424e691f47f64a1db3ef9b7cc40f8619 is 1080, key is row0254/info:/1733188670551/Put/seqid=0
2024-12-03T01:17:52,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741875_1051 (size=8199)
2024-12-03T01:17:52,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741875_1051 (size=8199)
2024-12-03T01:17:52,629 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/424e691f47f64a1db3ef9b7cc40f8619
2024-12-03T01:17:52,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/.tmp/info/424e691f47f64a1db3ef9b7cc40f8619 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/424e691f47f64a1db3ef9b7cc40f8619
2024-12-03T01:17:52,639 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/424e691f47f64a1db3ef9b7cc40f8619, entries=3, sequenceid=326, filesize=8.0 K
2024-12-03T01:17:52,640 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d47329c78b19d07e1df6eca553401e59 in 29ms, sequenceid=326, compaction requested=false
2024-12-03T01:17:52,640 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:52,640 INFO  [Time-limited test {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=2.21 KB heapSize=4.13 KB
2024-12-03T01:17:52,647 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/5c0edac72bf34c31be0343c6acb337ce is 193, key is TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59./info:regioninfo/1733188580803/Put/seqid=0
2024-12-03T01:17:52,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741876_1052 (size=7803)
2024-12-03T01:17:52,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741876_1052 (size=7803)
2024-12-03T01:17:52,659 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.21 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/5c0edac72bf34c31be0343c6acb337ce
2024-12-03T01:17:52,665 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/.tmp/info/5c0edac72bf34c31be0343c6acb337ce as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/info/5c0edac72bf34c31be0343c6acb337ce
2024-12-03T01:17:52,670 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/info/5c0edac72bf34c31be0343c6acb337ce, entries=16, sequenceid=24, filesize=7.6 K
2024-12-03T01:17:52,671 INFO  [Time-limited test {}] regionserver.HRegion(3040): Finished flush of dataSize ~2.21 KB/2260, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=24, compaction requested=false
2024-12-03T01:17:52,671 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for 1588230740:

2024-12-03T01:17:52,671 DEBUG [Time-limited test {}] regionserver.HRegion(2538): Flush status journal for c1d6cf602b426330b7916c2e5de4ab86:

2024-12-03T01:17:52,671 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C45383%2C1733188544276.1733188672671
2024-12-03T01:17:52,681 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188672554 with entries=4, filesize=1.22 KB; new WAL /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188672671
2024-12-03T01:17:52,682 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33535:33535),(127.0.0.1/127.0.0.1:45751:45751)]
2024-12-03T01:17:52,682 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188672554 is not closed yet, will try archiving it next time
2024-12-03T01:17:52,682 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188544803 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs/1a7b1a4db65e%2C45383%2C1733188544276.1733188544803
2024-12-03T01:17:52,682 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-03T01:17:52,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741873_1049 (size=1255)
2024-12-03T01:17:52,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741873_1049 (size=1255)
2024-12-03T01:17:52,684 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276/1a7b1a4db65e%2C45383%2C1733188544276.1733188672554 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs/1a7b1a4db65e%2C45383%2C1733188544276.1733188672554
2024-12-03T01:17:52,783 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:17:52,783 INFO  [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService
2024-12-03T01:17:52,783 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x400591a6 to 127.0.0.1:65274
2024-12-03T01:17:52,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:52,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:17:52,783 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1873158812, stopped=false
2024-12-03T01:17:52,783 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,44017,1733188544172
2024-12-03T01:17:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:52,785 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:17:52,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:52,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:52,785 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,45383,1733188544276' *****
2024-12-03T01:17:52,785 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:17:52,785 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:17:52,786 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:17:52,786 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(3579): Received CLOSE for 97f43709f8e893c79b431e7315d66f1f
2024-12-03T01:17:52,786 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(3579): Received CLOSE for d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(3579): Received CLOSE for c1d6cf602b426330b7916c2e5de4ab86
2024-12-03T01:17:52,786 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 97f43709f8e893c79b431e7315d66f1f, disabling compactions & flushes
2024-12-03T01:17:52,786 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:52,786 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:17:52,786 DEBUG [RS:0;1a7b1a4db65e:45383 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:52,786 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:17:52,786 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f. after waiting 0 ms
2024-12-03T01:17:52,786 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:17:52,787 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:17:52,787 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:17:52,787 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:17:52,787 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:17:52,787 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1599): Waiting on 4 regions to close
2024-12-03T01:17:52,787 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1603): Online Regions={97f43709f8e893c79b431e7315d66f1f=hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f., d47329c78b19d07e1df6eca553401e59=TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59., 1588230740=hbase:meta,,1.1588230740, c1d6cf602b426330b7916c2e5de4ab86=TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.}
2024-12-03T01:17:52,787 DEBUG [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 97f43709f8e893c79b431e7315d66f1f, c1d6cf602b426330b7916c2e5de4ab86, d47329c78b19d07e1df6eca553401e59
2024-12-03T01:17:52,787 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:17:52,787 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:17:52,787 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:17:52,787 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:17:52,787 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:17:52,791 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/namespace/97f43709f8e893c79b431e7315d66f1f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T01:17:52,791 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/hbase/meta/1588230740/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1
2024-12-03T01:17:52,792 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 97f43709f8e893c79b431e7315d66f1f:

2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T01:17:52,792 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733188545233.97f43709f8e893c79b431e7315d66f1f.
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing d47329c78b19d07e1df6eca553401e59, disabling compactions & flushes
2024-12-03T01:17:52,792 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:52,792 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59. after waiting 0 ms
2024-12-03T01:17:52,793 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:52,793 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/fc0eac3466c440248de9fc4f1fb4882c, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb] to archive
2024-12-03T01:17:52,793 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-12-03T01:17:52,795 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/fc0eac3466c440248de9fc4f1fb4882c to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/fc0eac3466c440248de9fc4f1fb4882c
2024-12-03T01:17:52,796 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29 to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/2d8b12285f8e4cb5a2b595043ac37e29
2024-12-03T01:17:52,798 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb to hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/archive/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/info/a8ce520720024d71b42ddd92d2ba47bb
2024-12-03T01:17:52,802 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/d47329c78b19d07e1df6eca553401e59/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=113
2024-12-03T01:17:52,803 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for d47329c78b19d07e1df6eca553401e59:

2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733188580096.d47329c78b19d07e1df6eca553401e59.
2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing c1d6cf602b426330b7916c2e5de4ab86, disabling compactions & flushes
2024-12-03T01:17:52,803 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86. after waiting 0 ms
2024-12-03T01:17:52,803 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:17:52,807 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/data/default/TestLogRolling-testLogRolling/c1d6cf602b426330b7916c2e5de4ab86/recovered.edits/118.seqid, newMaxSeqId=118, maxSeqId=113
2024-12-03T01:17:52,808 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:17:52,808 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for c1d6cf602b426330b7916c2e5de4ab86:

2024-12-03T01:17:52,808 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733188580096.c1d6cf602b426330b7916c2e5de4ab86.
2024-12-03T01:17:52,987 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,45383,1733188544276; all regions closed.
2024-12-03T01:17:52,988 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:52,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741834_1010 (size=9351)
2024-12-03T01:17:52,990 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741834_1010 (size=9351)
2024-12-03T01:17:52,992 DEBUG [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs
2024-12-03T01:17:52,992 INFO  [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C45383%2C1733188544276.meta:.meta(num 1733188545190)
2024-12-03T01:17:52,992 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/WALs/1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:52,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741877_1053 (size=1071)
2024-12-03T01:17:52,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741877_1053 (size=1071)
2024-12-03T01:17:52,996 DEBUG [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/oldWALs
2024-12-03T01:17:52,996 INFO  [RS:0;1a7b1a4db65e:45383 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C45383%2C1733188544276:(num 1733188672671)
2024-12-03T01:17:52,996 DEBUG [RS:0;1a7b1a4db65e:45383 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:52,996 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:17:52,997 INFO  [RS:0;1a7b1a4db65e:45383 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-03T01:17:52,997 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:17:52,997 INFO  [RS:0;1a7b1a4db65e:45383 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45383
2024-12-03T01:17:52,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,45383,1733188544276
2024-12-03T01:17:52,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:17:53,001 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,45383,1733188544276]
2024-12-03T01:17:53,001 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,45383,1733188544276; numProcessing=1
2024-12-03T01:17:53,003 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,45383,1733188544276 already deleted, retry=false
2024-12-03T01:17:53,003 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,45383,1733188544276 expired; onlineServers=0
2024-12-03T01:17:53,004 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,44017,1733188544172' *****
2024-12-03T01:17:53,004 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:17:53,004 DEBUG [M:0;1a7b1a4db65e:44017 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30302354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:17:53,004 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,44017,1733188544172
2024-12-03T01:17:53,004 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,44017,1733188544172; all regions closed.
2024-12-03T01:17:53,004 DEBUG [M:0;1a7b1a4db65e:44017 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:53,004 DEBUG [M:0;1a7b1a4db65e:44017 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:17:53,004 DEBUG [M:0;1a7b1a4db65e:44017 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:17:53,004 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:17:53,004 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188544498 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188544498,5,FailOnTimeoutGroup]
2024-12-03T01:17:53,004 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188544499 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188544499,5,FailOnTimeoutGroup]
2024-12-03T01:17:53,004 INFO  [M:0;1a7b1a4db65e:44017 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:17:53,005 DEBUG [M:0;1a7b1a4db65e:44017 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:17:53,005 INFO  [M:0;1a7b1a4db65e:44017 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:17:53,005 INFO  [M:0;1a7b1a4db65e:44017 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:17:53,005 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:17:53,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:17:53,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:53,005 DEBUG [M:0;1a7b1a4db65e:44017 {}] zookeeper.ZKUtil(347): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:17:53,005 WARN  [M:0;1a7b1a4db65e:44017 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:17:53,005 INFO  [M:0;1a7b1a4db65e:44017 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:17:53,005 INFO  [M:0;1a7b1a4db65e:44017 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:17:53,006 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:17:53,006 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:53,006 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:53,006 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:17:53,006 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:53,006 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:17:53,006 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=66.44 KB heapSize=81.69 KB
2024-12-03T01:17:53,023 DEBUG [M:0;1a7b1a4db65e:44017 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cff32ff7c8c54e319a2632808975f6f5 is 82, key is hbase:meta,,1/info:regioninfo/1733188545211/Put/seqid=0
2024-12-03T01:17:53,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741878_1054 (size=5672)
2024-12-03T01:17:53,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741878_1054 (size=5672)
2024-12-03T01:17:53,029 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cff32ff7c8c54e319a2632808975f6f5
2024-12-03T01:17:53,055 DEBUG [M:0;1a7b1a4db65e:44017 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0069001ac062496abc203ad17a03456c is 750, key is \x00\x00\x00\x00\x00\x00\x00\x09/proc:d/1733188546158/Put/seqid=0
2024-12-03T01:17:53,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741879_1055 (size=7277)
2024-12-03T01:17:53,060 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741879_1055 (size=7277)
2024-12-03T01:17:53,060 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65.83 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0069001ac062496abc203ad17a03456c
2024-12-03T01:17:53,064 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0069001ac062496abc203ad17a03456c
2024-12-03T01:17:53,078 DEBUG [M:0;1a7b1a4db65e:44017 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aaf0a38013094d0f8251f7ea2f9a3679 is 69, key is 1a7b1a4db65e,45383,1733188544276/rs:state/1733188544644/Put/seqid=0
2024-12-03T01:17:53,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741880_1056 (size=5156)
2024-12-03T01:17:53,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741880_1056 (size=5156)
2024-12-03T01:17:53,083 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aaf0a38013094d0f8251f7ea2f9a3679
2024-12-03T01:17:53,101 DEBUG [M:0;1a7b1a4db65e:44017 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0a9491658eae41469bfc2d4cab5112c5 is 52, key is load_balancer_on/state:d/1733188545711/Put/seqid=0
2024-12-03T01:17:53,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:53,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x10175e450f80001, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:53,101 INFO  [RS:0;1a7b1a4db65e:45383 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,45383,1733188544276; zookeeper connection closed.
2024-12-03T01:17:53,101 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6162f00b {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6162f00b
2024-12-03T01:17:53,102 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T01:17:53,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741881_1057 (size=5056)
2024-12-03T01:17:53,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741881_1057 (size=5056)
2024-12-03T01:17:53,106 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0a9491658eae41469bfc2d4cab5112c5
2024-12-03T01:17:53,111 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cff32ff7c8c54e319a2632808975f6f5 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cff32ff7c8c54e319a2632808975f6f5
2024-12-03T01:17:53,116 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cff32ff7c8c54e319a2632808975f6f5, entries=8, sequenceid=164, filesize=5.5 K
2024-12-03T01:17:53,116 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:53,117 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0069001ac062496abc203ad17a03456c as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0069001ac062496abc203ad17a03456c
2024-12-03T01:17:53,122 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0069001ac062496abc203ad17a03456c
2024-12-03T01:17:53,122 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0069001ac062496abc203ad17a03456c, entries=18, sequenceid=164, filesize=7.1 K
2024-12-03T01:17:53,123 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aaf0a38013094d0f8251f7ea2f9a3679 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aaf0a38013094d0f8251f7ea2f9a3679
2024-12-03T01:17:53,127 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aaf0a38013094d0f8251f7ea2f9a3679, entries=1, sequenceid=164, filesize=5.0 K
2024-12-03T01:17:53,128 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0a9491658eae41469bfc2d4cab5112c5 as hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0a9491658eae41469bfc2d4cab5112c5
2024-12-03T01:17:53,132 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:34085/user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0a9491658eae41469bfc2d4cab5112c5, entries=1, sequenceid=164, filesize=4.9 K
2024-12-03T01:17:53,133 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(3040): Finished flush of dataSize ~66.44 KB/68031, heapSize ~81.63 KB/83584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=164, compaction requested=false
2024-12-03T01:17:53,135 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:53,135 DEBUG [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:17:53,135 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/e19bfe31-29fe-2664-bcdc-7d24e39b418e/MasterData/WALs/1a7b1a4db65e,44017,1733188544172
2024-12-03T01:17:53,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46169 is added to blk_1073741830_1006 (size=79260)
2024-12-03T01:17:53,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35921 is added to blk_1073741830_1006 (size=79260)
2024-12-03T01:17:53,137 INFO  [M:0;1a7b1a4db65e:44017 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:17:53,137 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:17:53,137 INFO  [M:0;1a7b1a4db65e:44017 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:44017
2024-12-03T01:17:53,140 DEBUG [M:0;1a7b1a4db65e:44017 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,44017,1733188544172 already deleted, retry=false
2024-12-03T01:17:53,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:53,242 INFO  [M:0;1a7b1a4db65e:44017 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,44017,1733188544172; zookeeper connection closed.
2024-12-03T01:17:53,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44017-0x10175e450f80000, quorum=127.0.0.1:65274, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:53,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9e1a806{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:53,245 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b90b736{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:53,245 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:53,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3daa8d32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:53,246 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@533ed053{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:53,247 WARN  [BP-1275478181-172.17.0.2-1733188543022 heartbeating to localhost/127.0.0.1:34085 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:17:53,247 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:17:53,247 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:17:53,247 WARN  [BP-1275478181-172.17.0.2-1733188543022 heartbeating to localhost/127.0.0.1:34085 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1275478181-172.17.0.2-1733188543022 (Datanode Uuid 449a28b7-b637-4546-a1db-fe493480ce77) service to localhost/127.0.0.1:34085
2024-12-03T01:17:53,248 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data3/current/BP-1275478181-172.17.0.2-1733188543022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:53,248 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data4/current/BP-1275478181-172.17.0.2-1733188543022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:53,248 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:17:53,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@734fa1f2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:53,251 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@24316e4c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:53,251 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:53,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1104cb6c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:53,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f3fedd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:53,252 WARN  [BP-1275478181-172.17.0.2-1733188543022 heartbeating to localhost/127.0.0.1:34085 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:17:53,252 WARN  [BP-1275478181-172.17.0.2-1733188543022 heartbeating to localhost/127.0.0.1:34085 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1275478181-172.17.0.2-1733188543022 (Datanode Uuid bc866916-f249-411f-a0c9-82690b8688f2) service to localhost/127.0.0.1:34085
2024-12-03T01:17:53,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:17:53,252 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:17:53,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data1/current/BP-1275478181-172.17.0.2-1733188543022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:53,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/cluster_ab6281e9-16f9-a648-7628-d321b2e6fb47/dfs/data/data2/current/BP-1275478181-172.17.0.2-1733188543022 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:53,253 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:17:53,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@72ecbbb0{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:17:53,260 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11832530{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:53,260 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:53,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e2b1db8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:53,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58c48638{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:53,270 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:17:53,296 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:17:53,303 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=121 (was 109) - Thread LEAK? -, OpenFileDescriptor=490 (was 464) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=2675 (was 2579) - AvailableMemoryMB LEAK? -
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=122, OpenFileDescriptor=490, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=2675
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.log.dir so I do NOT create it in target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dfcee134-6552-e340-ec24-494b426c30f2/hadoop.tmp.dir so I do NOT create it in target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0, deleteOnExit=true
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/test.cache.data in system properties and HBase conf
2024-12-03T01:17:53,309 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.tmp.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF
2024-12-03T01:17:53,310 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/nfs.dump.dir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/java.io.tmpdir in system properties and HBase conf
2024-12-03T01:17:53,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-03T01:17:53,311 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-03T01:17:53,311 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-03T01:17:53,324 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:17:53,437 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:17:53,442 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:17:53,444 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:17:53,444 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:17:53,444 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:17:53,445 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:17:53,446 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e87a57b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:17:53,446 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7593038f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:17:53,585 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ff94620{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/java.io.tmpdir/jetty-localhost-38775-hadoop-hdfs-3_4_1-tests_jar-_-any-5578723065462357242/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:17:53,585 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@525e42b4{HTTP/1.1, (http/1.1)}{localhost:38775}
2024-12-03T01:17:53,585 INFO  [Time-limited test {}] server.Server(415): Started @421585ms
2024-12-03T01:17:53,598 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-03T01:17:53,662 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:17:53,665 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:17:53,665 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:17:53,665 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:17:53,665 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-03T01:17:53,666 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31913cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:17:53,666 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fd4cd29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:17:53,781 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16d4fe83{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/java.io.tmpdir/jetty-localhost-45521-hadoop-hdfs-3_4_1-tests_jar-_-any-6436799694591862756/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:53,781 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@69478c75{HTTP/1.1, (http/1.1)}{localhost:45521}
2024-12-03T01:17:53,781 INFO  [Time-limited test {}] server.Server(415): Started @421781ms
2024-12-03T01:17:53,782 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:17:53,812 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-03T01:17:53,815 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-03T01:17:53,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-03T01:17:53,816 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-03T01:17:53,816 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-03T01:17:53,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46532079{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,AVAILABLE}
2024-12-03T01:17:53,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3385e1f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-03T01:17:53,889 WARN  [Thread-2216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data1/current/BP-632156292-172.17.0.2-1733188673340/current, will proceed with Du for space computation calculation, 
2024-12-03T01:17:53,889 WARN  [Thread-2217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data2/current/BP-632156292-172.17.0.2-1733188673340/current, will proceed with Du for space computation calculation, 
2024-12-03T01:17:53,906 WARN  [Thread-2195 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:17:53,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x31697b5329709852 with lease ID 0x6b6baca3592d3ae8: Processing first storage report for DS-7907508e-84a7-4d36-bc27-fa193a5667fe from datanode DatanodeRegistration(127.0.0.1:42955, datanodeUuid=7d20816d-ad40-460e-a867-8a610ce14008, infoPort=33879, infoSecurePort=0, ipcPort=35191, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340)
2024-12-03T01:17:53,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x31697b5329709852 with lease ID 0x6b6baca3592d3ae8: from storage DS-7907508e-84a7-4d36-bc27-fa193a5667fe node DatanodeRegistration(127.0.0.1:42955, datanodeUuid=7d20816d-ad40-460e-a867-8a610ce14008, infoPort=33879, infoSecurePort=0, ipcPort=35191, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:17:53,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x31697b5329709852 with lease ID 0x6b6baca3592d3ae8: Processing first storage report for DS-a3a1c1fb-759c-42d3-ad17-c6b402436294 from datanode DatanodeRegistration(127.0.0.1:42955, datanodeUuid=7d20816d-ad40-460e-a867-8a610ce14008, infoPort=33879, infoSecurePort=0, ipcPort=35191, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340)
2024-12-03T01:17:53,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x31697b5329709852 with lease ID 0x6b6baca3592d3ae8: from storage DS-a3a1c1fb-759c-42d3-ad17-c6b402436294 node DatanodeRegistration(127.0.0.1:42955, datanodeUuid=7d20816d-ad40-460e-a867-8a610ce14008, infoPort=33879, infoSecurePort=0, ipcPort=35191, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:17:53,937 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49175ffa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/java.io.tmpdir/jetty-localhost-33601-hadoop-hdfs-3_4_1-tests_jar-_-any-8063723887634502560/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:53,937 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78d17fa0{HTTP/1.1, (http/1.1)}{localhost:33601}
2024-12-03T01:17:53,937 INFO  [Time-limited test {}] server.Server(415): Started @421937ms
2024-12-03T01:17:53,938 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-03T01:17:54,056 WARN  [Thread-2242 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data3/current/BP-632156292-172.17.0.2-1733188673340/current, will proceed with Du for space computation calculation, 
2024-12-03T01:17:54,056 WARN  [Thread-2243 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data4/current/BP-632156292-172.17.0.2-1733188673340/current, will proceed with Du for space computation calculation, 
2024-12-03T01:17:54,079 WARN  [Thread-2231 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-03T01:17:54,082 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec18a056c09786e3 with lease ID 0x6b6baca3592d3ae9: Processing first storage report for DS-4f8bc6fc-efac-448b-9440-b400a3dd01c5 from datanode DatanodeRegistration(127.0.0.1:36731, datanodeUuid=2e177e28-f65e-4130-8588-ac85c267dd81, infoPort=38773, infoSecurePort=0, ipcPort=44767, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340)
2024-12-03T01:17:54,082 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec18a056c09786e3 with lease ID 0x6b6baca3592d3ae9: from storage DS-4f8bc6fc-efac-448b-9440-b400a3dd01c5 node DatanodeRegistration(127.0.0.1:36731, datanodeUuid=2e177e28-f65e-4130-8588-ac85c267dd81, infoPort=38773, infoSecurePort=0, ipcPort=44767, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-03T01:17:54,082 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec18a056c09786e3 with lease ID 0x6b6baca3592d3ae9: Processing first storage report for DS-8e55621b-3051-4551-85e4-ee2d5a03ebd6 from datanode DatanodeRegistration(127.0.0.1:36731, datanodeUuid=2e177e28-f65e-4130-8588-ac85c267dd81, infoPort=38773, infoSecurePort=0, ipcPort=44767, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340)
2024-12-03T01:17:54,082 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec18a056c09786e3 with lease ID 0x6b6baca3592d3ae9: from storage DS-8e55621b-3051-4551-85e4-ee2d5a03ebd6 node DatanodeRegistration(127.0.0.1:36731, datanodeUuid=2e177e28-f65e-4130-8588-ac85c267dd81, infoPort=38773, infoSecurePort=0, ipcPort=44767, storageInfo=lv=-57;cid=testClusterID;nsid=2090780778;c=1733188673340), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-03T01:17:54,116 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:54,161 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22
2024-12-03T01:17:54,164 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/zookeeper_0, clientPort=59150, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-03T01:17:54,165 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59150
2024-12-03T01:17:54,165 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,166 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:17:54,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741825_1001 (size=7)
2024-12-03T01:17:54,178 INFO  [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296 with version=8
2024-12-03T01:17:54,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1462): The hbase.fs.tmp.dir is set to hdfs://localhost:45713/user/jenkins/test-data/585896cc-d46e-0c0f-5217-80668afce329/hbase-staging
2024-12-03T01:17:54,180 INFO  [Time-limited test {}] client.ConnectionUtils(129): master/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:17:54,180 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,180 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,180 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:17:54,180 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:17:54,181 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:17:54,181 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:17:54,182 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38801
2024-12-03T01:17:54,183 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,184 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,186 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38801 connecting to ZooKeeper ensemble=127.0.0.1:59150
2024-12-03T01:17:54,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388010x0, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:17:54,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38801-0x10175e64d020000 connected
2024-12-03T01:17:54,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:17:54,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:54,214 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:17:54,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38801
2024-12-03T01:17:54,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38801
2024-12-03T01:17:54,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38801
2024-12-03T01:17:54,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38801
2024-12-03T01:17:54,221 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38801
2024-12-03T01:17:54,223 INFO  [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296, hbase.cluster.distributed=false
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] client.ConnectionUtils(129): regionserver/1a7b1a4db65e:0 server-side Connection retries=45
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-03T01:17:54,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-03T01:17:54,239 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-03T01:17:54,239 INFO  [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-03T01:17:54,241 INFO  [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:43663
2024-12-03T01:17:54,241 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-03T01:17:54,243 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-03T01:17:54,243 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,245 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,247 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:43663 connecting to ZooKeeper ensemble=127.0.0.1:59150
2024-12-03T01:17:54,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436630x0, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-03T01:17:54,249 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43663-0x10175e64d020001 connected
2024-12-03T01:17:54,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:17:54,250 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:54,250 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-03T01:17:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43663
2024-12-03T01:17:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43663
2024-12-03T01:17:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43663
2024-12-03T01:17:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43663
2024-12-03T01:17:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43663
2024-12-03T01:17:54,252 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:17:54,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:17:54,254 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:17:54,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,256 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:17:54,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-03T01:17:54,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,256 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1a7b1a4db65e,38801,1733188674180 from backup master directory
2024-12-03T01:17:54,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:17:54,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-03T01:17:54,259 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:17:54,259 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-03T01:17:54,265 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1a7b1a4db65e:38801
2024-12-03T01:17:54,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:17:54,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741826_1002 (size=42)
2024-12-03T01:17:54,271 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/hbase.id with ID: 168df816-b038-482e-8503-97e17d1f3c86
2024-12-03T01:17:54,283 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:17:54,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741827_1003 (size=196)
2024-12-03T01:17:54,300 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-03T01:17:54,301 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-03T01:17:54,301 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:17:54,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:17:54,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741828_1004 (size=1189)
2024-12-03T01:17:54,312 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store
2024-12-03T01:17:54,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:17:54,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741829_1005 (size=34)
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:17:54,320 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:54,320 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:54,320 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:17:54,321 WARN  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/.initializing
2024-12-03T01:17:54,321 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/WALs/1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,323 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C38801%2C1733188674180, suffix=, logDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/WALs/1a7b1a4db65e,38801,1733188674180, archiveDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/oldWALs, maxLogs=10
2024-12-03T01:17:54,324 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C38801%2C1733188674180.1733188674324
2024-12-03T01:17:54,333 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/WALs/1a7b1a4db65e,38801,1733188674180/1a7b1a4db65e%2C38801%2C1733188674180.1733188674324
2024-12-03T01:17:54,333 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38773:38773),(127.0.0.1/127.0.0.1:33879:33879)]
2024-12-03T01:17:54,333 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:17:54,334 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:54,334 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:17:54,334 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:17:54,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:17:54,338 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-03T01:17:54,338 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:54,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:17:54,340 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-03T01:17:54,340 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,340 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:17:54,341 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:17:54,342 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-03T01:17:54,342 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,342 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:17:54,342 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-03T01:17:54,343 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-03T01:17:54,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,343 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:17:54,344 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:17:54,344 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:17:54,346 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-03T01:17:54,347 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-03T01:17:54,349 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:17:54,349 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724732, jitterRate=-0.07845614850521088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-03T01:17:54,350 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:17:54,351 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-03T01:17:54,354 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732a35ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:17:54,354 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating...
2024-12-03T01:17:54,354 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-03T01:17:54,354 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-03T01:17:54,354 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-03T01:17:54,355 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 0 msec
2024-12-03T01:17:54,355 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 0 msec
2024-12-03T01:17:54,355 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-03T01:17:54,357 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-03T01:17:54,358 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-03T01:17:54,359 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false
2024-12-03T01:17:54,359 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-03T01:17:54,360 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-03T01:17:54,361 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false
2024-12-03T01:17:54,361 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-03T01:17:54,362 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-03T01:17:54,364 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false
2024-12-03T01:17:54,365 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-03T01:17:54,365 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false
2024-12-03T01:17:54,367 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-03T01:17:54,368 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-03T01:17:54,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:54,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:54,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,370 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=1a7b1a4db65e,38801,1733188674180, sessionid=0x10175e64d020000, setting cluster-up flag (Was=false)
2024-12-03T01:17:54,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,379 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-03T01:17:54,379 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,389 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-03T01:17:54,390 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:54,392 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta
2024-12-03T01:17:54,392 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2
2024-12-03T01:17:54,392 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1a7b1a4db65e,38801,1733188674180
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=5, maxPoolSize=5
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1a7b1a4db65e:0, corePoolSize=10, maxPoolSize=10
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:17:54,393 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,396 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:17:54,396 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733188704397
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-03T01:17:54,397 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-03T01:17:54,397 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-03T01:17:54,397 INFO  [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:17:54,400 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,400 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-03T01:17:54,401 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-03T01:17:54,401 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-03T01:17:54,403 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-03T01:17:54,403 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-03T01:17:54,404 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188674403,5,FailOnTimeoutGroup]
2024-12-03T01:17:54,404 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188674404,5,FailOnTimeoutGroup]
2024-12-03T01:17:54,404 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,404 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-03T01:17:54,404 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,404 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:17:54,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741831_1007 (size=1039)
2024-12-03T01:17:54,408 INFO  [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039
2024-12-03T01:17:54,408 INFO  [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296
2024-12-03T01:17:54,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:17:54,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741832_1008 (size=32)
2024-12-03T01:17:54,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:54,418 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:17:54,419 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:17:54,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,420 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:54,420 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:17:54,421 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:17:54,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:54,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:17:54,422 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:17:54,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:54,422 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:54,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740
2024-12-03T01:17:54,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740
2024-12-03T01:17:54,425 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:17:54,426 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:17:54,427 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:17:54,428 INFO  [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870246, jitterRate=0.10657566785812378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:17:54,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:17:54,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:17:54,428 INFO  [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:17:54,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:17:54,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:17:54,428 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:17:54,429 INFO  [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:17:54,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:17:54,430 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta
2024-12-03T01:17:54,430 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta
2024-12-03T01:17:54,430 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-03T01:17:54,431 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-03T01:17:54,432 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-03T01:17:54,464 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1a7b1a4db65e:43663
2024-12-03T01:17:54,465 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1008): ClusterId : 168df816-b038-482e-8503-97e17d1f3c86
2024-12-03T01:17:54,465 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-03T01:17:54,467 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-03T01:17:54,467 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-03T01:17:54,469 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-03T01:17:54,469 DEBUG [RS:0;1a7b1a4db65e:43663 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608ad60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:17:54,469 DEBUG [RS:0;1a7b1a4db65e:43663 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a41e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:17:54,469 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled
2024-12-03T01:17:54,469 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled
2024-12-03T01:17:54,469 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1090): About to register with Master.
2024-12-03T01:17:54,470 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(3073): reportForDuty to master=1a7b1a4db65e,38801,1733188674180 with isa=1a7b1a4db65e/172.17.0.2:43663, startcode=1733188674238
2024-12-03T01:17:54,470 DEBUG [RS:0;1a7b1a4db65e:43663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-03T01:17:54,472 INFO  [RS-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47751, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-03T01:17:54,472 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38801 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,472 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38801 {}] master.ServerManager(486): Registering regionserver=1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,473 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296
2024-12-03T01:17:54,473 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:41121
2024-12-03T01:17:54,473 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1
2024-12-03T01:17:54,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:17:54,477 DEBUG [RS:0;1a7b1a4db65e:43663 {}] zookeeper.ZKUtil(111): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,477 WARN  [RS:0;1a7b1a4db65e:43663 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-03T01:17:54,477 INFO  [RS:0;1a7b1a4db65e:43663 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:17:54,477 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,477 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1a7b1a4db65e,43663,1733188674238]
2024-12-03T01:17:54,479 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds
2024-12-03T01:17:54,480 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-03T01:17:54,481 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-03T01:17:54,481 INFO  [RS:0;1a7b1a4db65e:43663 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-03T01:17:54,481 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,481 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S
2024-12-03T01:17:54,482 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=2, maxPoolSize=2
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,482 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,483 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,483 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1a7b1a4db65e:0, corePoolSize=1, maxPoolSize=1
2024-12-03T01:17:54,483 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:17:54,483 DEBUG [RS:0;1a7b1a4db65e:43663 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1a7b1a4db65e:0, corePoolSize=3, maxPoolSize=3
2024-12-03T01:17:54,483 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,483 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,483 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,483 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,483 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,43663,1733188674238-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:17:54,498 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-03T01:17:54,498 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,43663,1733188674238-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:54,512 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.Replication(204): 1a7b1a4db65e,43663,1733188674238 started
2024-12-03T01:17:54,512 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1767): Serving as 1a7b1a4db65e,43663,1733188674238, RpcServer on 1a7b1a4db65e/172.17.0.2:43663, sessionid=0x10175e64d020001
2024-12-03T01:17:54,512 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-03T01:17:54,512 DEBUG [RS:0;1a7b1a4db65e:43663 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,512 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,43663,1733188674238'
2024-12-03T01:17:54,512 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1a7b1a4db65e,43663,1733188674238'
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-03T01:17:54,513 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-03T01:17:54,514 DEBUG [RS:0;1a7b1a4db65e:43663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-03T01:17:54,514 INFO  [RS:0;1a7b1a4db65e:43663 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-03T01:17:54,514 INFO  [RS:0;1a7b1a4db65e:43663 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-03T01:17:54,582 WARN  [1a7b1a4db65e:38801 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions.
2024-12-03T01:17:54,616 INFO  [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C43663%2C1733188674238, suffix=, logDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238, archiveDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs, maxLogs=32
2024-12-03T01:17:54,616 INFO  [RS:0;1a7b1a4db65e:43663 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43663%2C1733188674238.1733188674616
2024-12-03T01:17:54,622 INFO  [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238/1a7b1a4db65e%2C43663%2C1733188674238.1733188674616
2024-12-03T01:17:54,623 DEBUG [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38773:38773),(127.0.0.1/127.0.0.1:33879:33879)]
2024-12-03T01:17:54,671 INFO  [regionserver/1a7b1a4db65e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:17:54,832 DEBUG [1a7b1a4db65e:38801 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-03T01:17:54,833 INFO  [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,834 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,43663,1733188674238, state=OPENING
2024-12-03T01:17:54,835 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-03T01:17:54,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:54,837 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,43663,1733188674238}]
2024-12-03T01:17:54,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:17:54,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:17:54,989 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:54,989 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-03T01:17:54,991 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43644, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-03T01:17:54,994 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740
2024-12-03T01:17:54,994 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:17:54,996 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1a7b1a4db65e%2C43663%2C1733188674238.meta, suffix=.meta, logDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238, archiveDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs, maxLogs=32
2024-12-03T01:17:54,997 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 1a7b1a4db65e%2C43663%2C1733188674238.meta.1733188674997.meta
2024-12-03T01:17:55,001 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238/1a7b1a4db65e%2C43663%2C1733188674238.meta.1733188674997.meta
2024-12-03T01:17:55,001 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33879:33879),(127.0.0.1/127.0.0.1:38773:38773)]
2024-12-03T01:17:55,001 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-03T01:17:55,002 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740
2024-12-03T01:17:55,002 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740
2024-12-03T01:17:55,003 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-03T01:17:55,004 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-03T01:17:55,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:55,005 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:55,005 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-03T01:17:55,005 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-03T01:17:55,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:55,006 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:55,006 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-03T01:17:55,006 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-03T01:17:55,007 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:55,007 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-03T01:17:55,007 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740
2024-12-03T01:17:55,008 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740
2024-12-03T01:17:55,010 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-03T01:17:55,011 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740
2024-12-03T01:17:55,011 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734220, jitterRate=-0.06639204919338226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-03T01:17:55,012 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740:

2024-12-03T01:17:55,012 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733188674989
2024-12-03T01:17:55,014 DEBUG [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-03T01:17:55,014 INFO  [RS_OPEN_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740
2024-12-03T01:17:55,014 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,015 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1a7b1a4db65e,43663,1733188674238, state=OPEN
2024-12-03T01:17:55,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:17:55,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-03T01:17:55,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:17:55,022 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-03T01:17:55,023 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2
2024-12-03T01:17:55,023 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=1a7b1a4db65e,43663,1733188674238 in 185 msec
2024-12-03T01:17:55,025 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1
2024-12-03T01:17:55,025 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 593 msec
2024-12-03T01:17:55,027 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 635 msec
2024-12-03T01:17:55,027 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733188675027, completionTime=-1
2024-12-03T01:17:55,027 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-03T01:17:55,027 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster...
2024-12-03T01:17:55,028 DEBUG [hconnection-0x14c795e7-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:17:55,029 INFO  [RS-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:17:55,029 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1
2024-12-03T01:17:55,029 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733188735029
2024-12-03T01:17:55,029 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733188795029
2024-12-03T01:17:55,029 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 2 msec
2024-12-03T01:17:55,033 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,033 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,033 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,034 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1a7b1a4db65e:38801, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,034 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,034 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating...
2024-12-03T01:17:55,034 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-03T01:17:55,035 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace
2024-12-03T01:17:55,035 DEBUG [master/1a7b1a4db65e:0.Chore.1 {}] janitor.CatalogJanitor(179): 
2024-12-03T01:17:55,035 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION
2024-12-03T01:17:55,035 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:55,036 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-03T01:17:55,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:17:55,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741835_1011 (size=358)
2024-12-03T01:17:55,045 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8b0028a4082ab0f7e5a58b7a46960456, NAME => 'hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296
2024-12-03T01:17:55,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:17:55,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741836_1012 (size=42)
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 8b0028a4082ab0f7e5a58b7a46960456, disabling compactions & flushes
2024-12-03T01:17:55,052 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456. after waiting 0 ms
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,052 INFO  [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,052 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8b0028a4082ab0f7e5a58b7a46960456:

2024-12-03T01:17:55,053 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META
2024-12-03T01:17:55,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733188675053"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733188675053"}]},"ts":"1733188675053"}
2024-12-03T01:17:55,056 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta.
2024-12-03T01:17:55,056 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-03T01:17:55,057 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188675056"}]},"ts":"1733188675056"}
2024-12-03T01:17:55,058 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta
2024-12-03T01:17:55,067 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8b0028a4082ab0f7e5a58b7a46960456, ASSIGN}]
2024-12-03T01:17:55,068 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8b0028a4082ab0f7e5a58b7a46960456, ASSIGN
2024-12-03T01:17:55,069 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8b0028a4082ab0f7e5a58b7a46960456, ASSIGN; state=OFFLINE, location=1a7b1a4db65e,43663,1733188674238; forceNewPlan=false, retain=false
2024-12-03T01:17:55,117 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:43249/user/jenkins/test-data/259a3365-502c-a309-c4a3-19e27431f354/WALs/1a7b1a4db65e,37929,1733188372050/1a7b1a4db65e%2C37929%2C1733188372050.meta.1733188373474.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor118.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.closeWriter(FSHLog.java:464) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.lambda$doReplaceWriter$0(FSHLog.java:402) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-03T01:17:55,219 INFO  [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8b0028a4082ab0f7e5a58b7a46960456, regionState=OPENING, regionLocation=1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,221 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8b0028a4082ab0f7e5a58b7a46960456, server=1a7b1a4db65e,43663,1733188674238}]
2024-12-03T01:17:55,373 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,376 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 8b0028a4082ab0f7e5a58b7a46960456, NAME => 'hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.', STARTKEY => '', ENDKEY => ''}
2024-12-03T01:17:55,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-03T01:17:55,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,377 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,378 INFO  [StoreOpener-8b0028a4082ab0f7e5a58b7a46960456-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8b0028a4082ab0f7e5a58b7a46960456 
2024-12-03T01:17:55,379 INFO  [StoreOpener-8b0028a4082ab0f7e5a58b7a46960456-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b0028a4082ab0f7e5a58b7a46960456 columnFamilyName info
2024-12-03T01:17:55,379 DEBUG [StoreOpener-8b0028a4082ab0f7e5a58b7a46960456-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-03T01:17:55,380 INFO  [StoreOpener-8b0028a4082ab0f7e5a58b7a46960456-1 {}] regionserver.HStore(327): Store=8b0028a4082ab0f7e5a58b7a46960456/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-03T01:17:55,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,381 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,383 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-03T01:17:55,386 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 8b0028a4082ab0f7e5a58b7a46960456; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715985, jitterRate=-0.08957812190055847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-03T01:17:55,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 8b0028a4082ab0f7e5a58b7a46960456:

2024-12-03T01:17:55,386 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456., pid=6, masterSystemTime=1733188675373
2024-12-03T01:17:55,388 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,388 INFO  [RS_OPEN_PRIORITY_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,389 INFO  [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8b0028a4082ab0f7e5a58b7a46960456, regionState=OPEN, openSeqNum=2, regionLocation=1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,392 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5
2024-12-03T01:17:55,392 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8b0028a4082ab0f7e5a58b7a46960456, server=1a7b1a4db65e,43663,1733188674238 in 169 msec
2024-12-03T01:17:55,394 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4
2024-12-03T01:17:55,394 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8b0028a4082ab0f7e5a58b7a46960456, ASSIGN in 325 msec
2024-12-03T01:17:55,395 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-03T01:17:55,395 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733188675395"}]},"ts":"1733188675395"}
2024-12-03T01:17:55,396 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta
2024-12-03T01:17:55,399 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION
2024-12-03T01:17:55,400 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 365 msec
2024-12-03T01:17:55,436 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace
2024-12-03T01:17:55,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:17:55,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:55,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:55,442 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default
2024-12-03T01:17:55,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:17:55,453 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 11 msec
2024-12-03T01:17:55,464 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase
2024-12-03T01:17:55,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace
2024-12-03T01:17:55,477 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 13 msec
2024-12-03T01:17:55,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default
2024-12-03T01:17:55,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 1.234sec
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-03T01:17:55,493 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-03T01:17:55,494 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds
2024-12-03T01:17:55,495 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-03T01:17:55,495 INFO  [master/1a7b1a4db65e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1a7b1a4db65e,38801,1733188674180-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-03T01:17:55,555 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f367d70 to 127.0.0.1:59150 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54a90a71
2024-12-03T01:17:55,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e09df6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-03T01:17:55,560 DEBUG [hconnection-0x15a8a0af-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-03T01:17:55,561 INFO  [RS-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43652, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-03T01:17:55,563 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:55,563 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-03T01:17:55,566 INFO  [Time-limited test {}] master.MasterRpcServices(506): Client=null/null set balanceSwitch=false
2024-12-03T01:17:55,566 INFO  [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-03T01:17:55,568 INFO  [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1, archiveDir=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs, maxLogs=32
2024-12-03T01:17:55,569 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733188675569
2024-12-03T01:17:55,574 INFO  [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1/test.com%2C8080%2C1.1733188675569
2024-12-03T01:17:55,574 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33879:33879),(127.0.0.1/127.0.0.1:38773:38773)]
2024-12-03T01:17:55,574 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733188675574
2024-12-03T01:17:55,583 INFO  [Time-limited test {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1/test.com%2C8080%2C1.1733188675569 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1/test.com%2C8080%2C1.1733188675574
2024-12-03T01:17:55,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38773:38773),(127.0.0.1/127.0.0.1:33879:33879)]
2024-12-03T01:17:55,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(751): hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1/test.com%2C8080%2C1.1733188675569 is not closed yet, will try archiving it next time
2024-12-03T01:17:55,585 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1
2024-12-03T01:17:55,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741837_1013 (size=93)
2024-12-03T01:17:55,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741838_1014 (size=93)
2024-12-03T01:17:55,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741837_1013 (size=93)
2024-12-03T01:17:55,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741838_1014 (size=93)
2024-12-03T01:17:55,587 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(818): Archiving hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/test.com,8080,1/test.com%2C8080%2C1.1733188675569 to hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs/test.com%2C8080%2C1.1733188675569
2024-12-03T01:17:55,589 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs
2024-12-03T01:17:55,590 INFO  [Time-limited test {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733188675574)
2024-12-03T01:17:55,590 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster
2024-12-03T01:17:55,590 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f367d70 to 127.0.0.1:59150
2024-12-03T01:17:55,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:55,590 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-03T01:17:55,590 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=723317967, stopped=false
2024-12-03T01:17:55,590 INFO  [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-03T01:17:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:55,594 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping
2024-12-03T01:17:55,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:55,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:55,595 INFO  [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,43663,1733188674238' *****
2024-12-03T01:17:55,595 INFO  [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested
2024-12-03T01:17:55,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-03T01:17:55,595 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(3579): Received CLOSE for 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,595 DEBUG [RS:0;1a7b1a4db65e:43663 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-03T01:17:55,595 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8b0028a4082ab0f7e5a58b7a46960456, disabling compactions & flushes
2024-12-03T01:17:55,596 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740
2024-12-03T01:17:55,596 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456. after waiting 0 ms
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,596 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8b0028a4082ab0f7e5a58b7a46960456 1/1 column families, dataSize=78 B heapSize=488 B
2024-12-03T01:17:55,596 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close
2024-12-03T01:17:55,596 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 8b0028a4082ab0f7e5a58b7a46960456=hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.}
2024-12-03T01:17:55,596 DEBUG [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8b0028a4082ab0f7e5a58b7a46960456
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes
2024-12-03T01:17:55,596 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-03T01:17:55,596 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740
2024-12-03T01:17:55,596 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB
2024-12-03T01:17:55,612 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/.tmp/info/43f00f6a7dfe4692b9bbf3bece293cba is 45, key is default/info:d/1733188675446/Put/seqid=0
2024-12-03T01:17:55,612 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/info/bcb57447f165460cace50c4d2c2da27f is 143, key is hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456./info:regioninfo/1733188675389/Put/seqid=0
2024-12-03T01:17:55,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741839_1015 (size=5037)
2024-12-03T01:17:55,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741840_1016 (size=6595)
2024-12-03T01:17:55,618 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/.tmp/info/43f00f6a7dfe4692b9bbf3bece293cba
2024-12-03T01:17:55,619 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/info/bcb57447f165460cace50c4d2c2da27f
2024-12-03T01:17:55,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741839_1015 (size=5037)
2024-12-03T01:17:55,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741840_1016 (size=6595)
2024-12-03T01:17:55,624 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/.tmp/info/43f00f6a7dfe4692b9bbf3bece293cba as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/info/43f00f6a7dfe4692b9bbf3bece293cba
2024-12-03T01:17:55,628 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/info/43f00f6a7dfe4692b9bbf3bece293cba, entries=2, sequenceid=6, filesize=4.9 K
2024-12-03T01:17:55,629 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8b0028a4082ab0f7e5a58b7a46960456 in 33ms, sequenceid=6, compaction requested=false
2024-12-03T01:17:55,633 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/namespace/8b0028a4082ab0f7e5a58b7a46960456/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-03T01:17:55,634 INFO  [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,634 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8b0028a4082ab0f7e5a58b7a46960456:

2024-12-03T01:17:55,634 DEBUG [RS_CLOSE_REGION-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733188675034.8b0028a4082ab0f7e5a58b7a46960456.
2024-12-03T01:17:55,639 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/table/af9f5b8c8d7f4bc489a6f81c612bfea8 is 51, key is hbase:namespace/table:state/1733188675395/Put/seqid=0
2024-12-03T01:17:55,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741841_1017 (size=5242)
2024-12-03T01:17:55,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741841_1017 (size=5242)
2024-12-03T01:17:55,644 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/table/af9f5b8c8d7f4bc489a6f81c612bfea8
2024-12-03T01:17:55,649 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/info/bcb57447f165460cace50c4d2c2da27f as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/info/bcb57447f165460cace50c4d2c2da27f
2024-12-03T01:17:55,653 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/info/bcb57447f165460cace50c4d2c2da27f, entries=10, sequenceid=9, filesize=6.4 K
2024-12-03T01:17:55,654 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/.tmp/table/af9f5b8c8d7f4bc489a6f81c612bfea8 as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/table/af9f5b8c8d7f4bc489a6f81c612bfea8
2024-12-03T01:17:55,658 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/table/af9f5b8c8d7f4bc489a6f81c612bfea8, entries=2, sequenceid=9, filesize=5.1 K
2024-12-03T01:17:55,659 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 63ms, sequenceid=9, compaction requested=false
2024-12-03T01:17:55,663 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1
2024-12-03T01:17:55,664 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-03T01:17:55,664 INFO  [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740
2024-12-03T01:17:55,664 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740:

2024-12-03T01:17:55,664 DEBUG [RS_CLOSE_META-regionserver/1a7b1a4db65e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-03T01:17:55,796 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,43663,1733188674238; all regions closed.
2024-12-03T01:17:55,797 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741834_1010 (size=2484)
2024-12-03T01:17:55,799 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741834_1010 (size=2484)
2024-12-03T01:17:55,801 DEBUG [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs
2024-12-03T01:17:55,801 INFO  [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C43663%2C1733188674238.meta:.meta(num 1733188674997)
2024-12-03T01:17:55,801 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/WALs/1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741833_1009 (size=1414)
2024-12-03T01:17:55,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741833_1009 (size=1414)
2024-12-03T01:17:55,804 DEBUG [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/oldWALs
2024-12-03T01:17:55,804 INFO  [RS:0;1a7b1a4db65e:43663 {}] wal.AbstractFSWAL(1074): Closed WAL: FSHLog 1a7b1a4db65e%2C43663%2C1733188674238:(num 1733188674616)
2024-12-03T01:17:55,804 DEBUG [RS:0;1a7b1a4db65e:43663 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:55,804 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.LeaseManager(133): Closed leases
2024-12-03T01:17:55,805 INFO  [RS:0;1a7b1a4db65e:43663 {}] hbase.ChoreService(370): Chore service for: regionserver/1a7b1a4db65e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown
2024-12-03T01:17:55,805 INFO  [regionserver/1a7b1a4db65e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:17:55,805 INFO  [RS:0;1a7b1a4db65e:43663 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:43663
2024-12-03T01:17:55,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1a7b1a4db65e,43663,1733188674238
2024-12-03T01:17:55,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-03T01:17:55,808 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1a7b1a4db65e,43663,1733188674238]
2024-12-03T01:17:55,808 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 1a7b1a4db65e,43663,1733188674238; numProcessing=1
2024-12-03T01:17:55,810 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/1a7b1a4db65e,43663,1733188674238 already deleted, retry=false
2024-12-03T01:17:55,811 INFO  [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 1a7b1a4db65e,43663,1733188674238 expired; onlineServers=0
2024-12-03T01:17:55,811 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '1a7b1a4db65e,38801,1733188674180' *****
2024-12-03T01:17:55,811 INFO  [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-03T01:17:55,811 DEBUG [M:0;1a7b1a4db65e:38801 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1a7b1a4db65e/172.17.0.2:0
2024-12-03T01:17:55,811 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionServer(1224): stopping server 1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:55,811 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionServer(1250): stopping server 1a7b1a4db65e,38801,1733188674180; all regions closed.
2024-12-03T01:17:55,811 DEBUG [M:0;1a7b1a4db65e:38801 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-03T01:17:55,811 DEBUG [M:0;1a7b1a4db65e:38801 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-03T01:17:55,811 DEBUG [M:0;1a7b1a4db65e:38801 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-03T01:17:55,811 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-03T01:17:55,811 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188674403 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.large.0-1733188674403,5,FailOnTimeoutGroup]
2024-12-03T01:17:55,811 DEBUG [master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188674404 {}] cleaner.HFileCleaner(306): Exit Thread[master/1a7b1a4db65e:0:becomeActiveMaster-HFileCleaner.small.0-1733188674404,5,FailOnTimeoutGroup]
2024-12-03T01:17:55,811 INFO  [M:0;1a7b1a4db65e:38801 {}] hbase.ChoreService(370): Chore service for: master/1a7b1a4db65e:0 had [] on shutdown
2024-12-03T01:17:55,811 DEBUG [M:0;1a7b1a4db65e:38801 {}] master.HMaster(1733): Stopping service threads
2024-12-03T01:17:55,811 INFO  [M:0;1a7b1a4db65e:38801 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-03T01:17:55,812 INFO  [M:0;1a7b1a4db65e:38801 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-03T01:17:55,812 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-03T01:17:55,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-03T01:17:55,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-03T01:17:55,812 DEBUG [M:0;1a7b1a4db65e:38801 {}] zookeeper.ZKUtil(347): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-03T01:17:55,812 WARN  [M:0;1a7b1a4db65e:38801 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-03T01:17:55,812 INFO  [M:0;1a7b1a4db65e:38801 {}] assignment.AssignmentManager(391): Stopping assignment manager
2024-12-03T01:17:55,812 INFO  [M:0;1a7b1a4db65e:38801 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-03T01:17:55,812 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-03T01:17:55,812 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:55,812 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-03T01:17:55,812 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:55,812 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-03T01:17:55,812 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:55,812 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=25.32 KB heapSize=32.31 KB
2024-12-03T01:17:55,828 DEBUG [M:0;1a7b1a4db65e:38801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5210cf7afc494ed68901b6225af828e0 is 82, key is hbase:meta,,1/info:regioninfo/1733188675014/Put/seqid=0
2024-12-03T01:17:55,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741842_1018 (size=5672)
2024-12-03T01:17:55,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741842_1018 (size=5672)
2024-12-03T01:17:55,833 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5210cf7afc494ed68901b6225af828e0
2024-12-03T01:17:55,851 DEBUG [M:0;1a7b1a4db65e:38801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00097657d0cd42baa5f3af80371eca18 is 696, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733188675400/Put/seqid=0
2024-12-03T01:17:55,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741843_1019 (size=6626)
2024-12-03T01:17:55,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741843_1019 (size=6626)
2024-12-03T01:17:55,857 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.72 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00097657d0cd42baa5f3af80371eca18
2024-12-03T01:17:55,877 DEBUG [M:0;1a7b1a4db65e:38801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2940df21bef4ae9a27942b3c2bb8f64 is 69, key is 1a7b1a4db65e,43663,1733188674238/rs:state/1733188674472/Put/seqid=0
2024-12-03T01:17:55,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741844_1020 (size=5156)
2024-12-03T01:17:55,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741844_1020 (size=5156)
2024-12-03T01:17:55,882 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2940df21bef4ae9a27942b3c2bb8f64
2024-12-03T01:17:55,907 DEBUG [M:0;1a7b1a4db65e:38801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/09a5ffdc19284744be3b624775e52644 is 52, key is load_balancer_on/state:d/1733188675565/Put/seqid=0
2024-12-03T01:17:55,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:55,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43663-0x10175e64d020001, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:55,908 INFO  [RS:0;1a7b1a4db65e:43663 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,43663,1733188674238; zookeeper connection closed.
2024-12-03T01:17:55,908 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d0411b9 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d0411b9
2024-12-03T01:17:55,909 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-03T01:17:55,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741845_1021 (size=5056)
2024-12-03T01:17:55,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741845_1021 (size=5056)
2024-12-03T01:17:55,913 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/09a5ffdc19284744be3b624775e52644
2024-12-03T01:17:55,917 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5210cf7afc494ed68901b6225af828e0 as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5210cf7afc494ed68901b6225af828e0
2024-12-03T01:17:55,921 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5210cf7afc494ed68901b6225af828e0, entries=8, sequenceid=70, filesize=5.5 K
2024-12-03T01:17:55,922 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00097657d0cd42baa5f3af80371eca18 as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00097657d0cd42baa5f3af80371eca18
2024-12-03T01:17:55,926 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00097657d0cd42baa5f3af80371eca18, entries=8, sequenceid=70, filesize=6.5 K
2024-12-03T01:17:55,926 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a2940df21bef4ae9a27942b3c2bb8f64 as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a2940df21bef4ae9a27942b3c2bb8f64
2024-12-03T01:17:55,930 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a2940df21bef4ae9a27942b3c2bb8f64, entries=1, sequenceid=70, filesize=5.0 K
2024-12-03T01:17:55,931 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/09a5ffdc19284744be3b624775e52644 as hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/09a5ffdc19284744be3b624775e52644
2024-12-03T01:17:55,935 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:41121/user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/09a5ffdc19284744be3b624775e52644, entries=1, sequenceid=70, filesize=4.9 K
2024-12-03T01:17:55,936 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(3040): Finished flush of dataSize ~25.32 KB/25929, heapSize ~32.25 KB/33024, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 124ms, sequenceid=70, compaction requested=false
2024-12-03T01:17:55,938 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-03T01:17:55,938 DEBUG [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682:

2024-12-03T01:17:55,938 DEBUG [WAL-Shutdown-0 {}] wal.FSHLog(499): Closing WAL writer in /user/jenkins/test-data/5c2197b5-8e3b-ebda-5852-a89b89131296/MasterData/WALs/1a7b1a4db65e,38801,1733188674180
2024-12-03T01:17:55,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42955 is added to blk_1073741830_1006 (size=31030)
2024-12-03T01:17:55,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36731 is added to blk_1073741830_1006 (size=31030)
2024-12-03T01:17:55,941 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting.
2024-12-03T01:17:55,941 INFO  [M:0;1a7b1a4db65e:38801 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down.
2024-12-03T01:17:55,941 INFO  [M:0;1a7b1a4db65e:38801 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38801
2024-12-03T01:17:55,943 DEBUG [M:0;1a7b1a4db65e:38801 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/1a7b1a4db65e,38801,1733188674180 already deleted, retry=false
2024-12-03T01:17:56,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:56,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38801-0x10175e64d020000, quorum=127.0.0.1:59150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-03T01:17:56,045 INFO  [M:0;1a7b1a4db65e:38801 {}] regionserver.HRegionServer(1307): Exiting; stopping=1a7b1a4db65e,38801,1733188674180; zookeeper connection closed.
2024-12-03T01:17:56,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49175ffa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:56,048 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78d17fa0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:56,048 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:56,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3385e1f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:56,048 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46532079{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:56,050 WARN  [BP-632156292-172.17.0.2-1733188673340 heartbeating to localhost/127.0.0.1:41121 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:17:56,050 WARN  [BP-632156292-172.17.0.2-1733188673340 heartbeating to localhost/127.0.0.1:41121 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-632156292-172.17.0.2-1733188673340 (Datanode Uuid 2e177e28-f65e-4130-8588-ac85c267dd81) service to localhost/127.0.0.1:41121
2024-12-03T01:17:56,050 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:17:56,050 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:17:56,050 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data3/current/BP-632156292-172.17.0.2-1733188673340 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:56,051 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data4/current/BP-632156292-172.17.0.2-1733188673340 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:56,051 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:17:56,053 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16d4fe83{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-03T01:17:56,053 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@69478c75{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:56,053 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:56,053 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fd4cd29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:56,053 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31913cb3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:56,054 WARN  [BP-632156292-172.17.0.2-1733188673340 heartbeating to localhost/127.0.0.1:41121 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-03T01:17:56,054 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-03T01:17:56,054 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-03T01:17:56,054 WARN  [BP-632156292-172.17.0.2-1733188673340 heartbeating to localhost/127.0.0.1:41121 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-632156292-172.17.0.2-1733188673340 (Datanode Uuid 7d20816d-ad40-460e-a867-8a610ce14008) service to localhost/127.0.0.1:41121
2024-12-03T01:17:56,055 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data1/current/BP-632156292-172.17.0.2-1733188673340 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:56,055 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/cluster_faafd16c-c408-7c90-d920-97b480b792c0/dfs/data/data2/current/BP-632156292-172.17.0.2-1733188673340 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-03T01:17:56,055 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-03T01:17:56,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ff94620{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-03T01:17:56,061 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@525e42b4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-03T01:17:56,061 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-03T01:17:56,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7593038f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-03T01:17:56,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e87a57b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/64b6cfc9-1179-cefe-c063-bfec4cbdeb22/hadoop.log.dir/,STOPPED}
2024-12-03T01:17:56,067 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers
2024-12-03T01:17:56,091 INFO  [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down
2024-12-03T01:17:56,100 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=145 (was 122) - Thread LEAK? -, OpenFileDescriptor=518 (was 490) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 226) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2620 (was 2675)