2024-11-22 17:33:34,788 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-22 17:33:34,804 main DEBUG Took 0.013434 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-22 17:33:34,804 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-22 17:33:34,805 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-22 17:33:34,806 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-22 17:33:34,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,816 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-22 17:33:34,834 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,837 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,838 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,839 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,840 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,842 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,843 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,844 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,846 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,848 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,850 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,851 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,852 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-22 17:33:34,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,853 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-22 17:33:34,855 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-22 17:33:34,856 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-22 17:33:34,859 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-22 17:33:34,860 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-22 17:33:34,862 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-22 17:33:34,863 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-22 17:33:34,874 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-22 17:33:34,877 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-22 17:33:34,880 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-22 17:33:34,881 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-22 17:33:34,881 main DEBUG createAppenders(={Console})
2024-11-22 17:33:34,882 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-22 17:33:34,883 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-22 17:33:34,883 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-22 17:33:34,884 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-22 17:33:34,884 main DEBUG OutputStream closed
2024-11-22 17:33:34,885 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-22 17:33:34,885 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-22 17:33:34,886 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-22 17:33:34,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-22 17:33:34,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-22 17:33:34,991 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-22 17:33:34,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-22 17:33:34,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-22 17:33:34,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-22 17:33:34,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-22 17:33:34,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-22 17:33:34,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-22 17:33:34,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-22 17:33:34,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-22 17:33:34,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-22 17:33:34,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-22 17:33:34,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-22 17:33:34,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-22 17:33:34,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-22 17:33:34,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-22 17:33:34,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-22 17:33:35,000 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22 17:33:35,000 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-22 17:33:35,001 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-22 17:33:35,002 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-22T17:33:35,308 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538
2024-11-22 17:33:35,314 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-22 17:33:35,314 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-22T17:33:35,327 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-22T17:33:35,374 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=376, ProcessCount=11, AvailableMemoryMB=3109
2024-11-22T17:33:35,378 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:33:35,408 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0, deleteOnExit=true
2024-11-22T17:33:35,409 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:33:35,413 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/test.cache.data in system properties and HBase conf
2024-11-22T17:33:35,414 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:33:35,415 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:33:35,416 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:33:35,418 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:33:35,418 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:33:35,548 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-22T17:33:35,684 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:33:35,694 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:33:35,695 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:33:35,696 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:33:35,697 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:33:35,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:33:35,699 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:33:35,699 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:33:35,700 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:33:35,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:33:35,702 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:33:35,703 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:33:35,703 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:33:35,704 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:33:35,705 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:33:36,222 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:33:37,035 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-22T17:33:37,119 INFO  [Time-limited test {}] log.Log(170): Logging initialized @3156ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-22T17:33:37,203 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:33:37,292 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:33:37,321 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:33:37,321 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:33:37,323 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:33:37,338 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:33:37,341 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:33:37,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:33:37,519 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de997b9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/java.io.tmpdir/jetty-localhost-39993-hadoop-hdfs-3_4_1-tests_jar-_-any-18152527293512854143/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:33:37,525 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:39993}
2024-11-22T17:33:37,525 INFO  [Time-limited test {}] server.Server(415): Started @3563ms
2024-11-22T17:33:37,553 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:33:38,115 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:33:38,122 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:33:38,123 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:33:38,124 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:33:38,124 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:33:38,125 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:33:38,125 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:33:38,229 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f93babe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/java.io.tmpdir/jetty-localhost-35109-hadoop-hdfs-3_4_1-tests_jar-_-any-11817961219944638460/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:33:38,229 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:35109}
2024-11-22T17:33:38,230 INFO  [Time-limited test {}] server.Server(415): Started @4267ms
2024-11-22T17:33:38,276 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:33:38,384 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:33:38,391 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:33:38,393 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:33:38,393 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:33:38,393 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:33:38,394 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:33:38,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:33:38,500 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c963ecd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/java.io.tmpdir/jetty-localhost-45725-hadoop-hdfs-3_4_1-tests_jar-_-any-11899918906041158495/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:33:38,501 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:45725}
2024-11-22T17:33:38,502 INFO  [Time-limited test {}] server.Server(415): Started @4539ms
2024-11-22T17:33:38,505 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:33:39,824 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data3/current/BP-200041043-172.17.0.2-1732296816317/current, will proceed with Du for space computation calculation, 
2024-11-22T17:33:39,824 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data1/current/BP-200041043-172.17.0.2-1732296816317/current, will proceed with Du for space computation calculation, 
2024-11-22T17:33:39,824 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data2/current/BP-200041043-172.17.0.2-1732296816317/current, will proceed with Du for space computation calculation, 
2024-11-22T17:33:39,824 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data4/current/BP-200041043-172.17.0.2-1732296816317/current, will proceed with Du for space computation calculation, 
2024-11-22T17:33:39,858 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:33:39,858 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:33:39,905 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4293a0c847b8cb12 with lease ID 0xe5743446d8887f07: Processing first storage report for DS-a8b3852b-ce26-4c79-be4a-d60da984df2f from datanode DatanodeRegistration(127.0.0.1:33349, datanodeUuid=d476f111-402c-449b-9df8-2f45e64954d8, infoPort=38499, infoSecurePort=0, ipcPort=38535, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317)
2024-11-22T17:33:39,907 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4293a0c847b8cb12 with lease ID 0xe5743446d8887f07: from storage DS-a8b3852b-ce26-4c79-be4a-d60da984df2f node DatanodeRegistration(127.0.0.1:33349, datanodeUuid=d476f111-402c-449b-9df8-2f45e64954d8, infoPort=38499, infoSecurePort=0, ipcPort=38535, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:33:39,907 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64835e7e50298446 with lease ID 0xe5743446d8887f08: Processing first storage report for DS-b07b3be2-f6d8-47ac-8f26-34457bdea851 from datanode DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b40d762e-a70a-4faa-95f1-2220c06987cb, infoPort=44459, infoSecurePort=0, ipcPort=43447, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317)
2024-11-22T17:33:39,907 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64835e7e50298446 with lease ID 0xe5743446d8887f08: from storage DS-b07b3be2-f6d8-47ac-8f26-34457bdea851 node DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b40d762e-a70a-4faa-95f1-2220c06987cb, infoPort=44459, infoSecurePort=0, ipcPort=43447, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:33:39,908 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4293a0c847b8cb12 with lease ID 0xe5743446d8887f07: Processing first storage report for DS-2bee5d4c-c17c-4347-a3fb-dbed75230b58 from datanode DatanodeRegistration(127.0.0.1:33349, datanodeUuid=d476f111-402c-449b-9df8-2f45e64954d8, infoPort=38499, infoSecurePort=0, ipcPort=38535, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317)
2024-11-22T17:33:39,908 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4293a0c847b8cb12 with lease ID 0xe5743446d8887f07: from storage DS-2bee5d4c-c17c-4347-a3fb-dbed75230b58 node DatanodeRegistration(127.0.0.1:33349, datanodeUuid=d476f111-402c-449b-9df8-2f45e64954d8, infoPort=38499, infoSecurePort=0, ipcPort=38535, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:33:39,908 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64835e7e50298446 with lease ID 0xe5743446d8887f08: Processing first storage report for DS-0c01abdd-98ed-4b12-b051-f60b47a52375 from datanode DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b40d762e-a70a-4faa-95f1-2220c06987cb, infoPort=44459, infoSecurePort=0, ipcPort=43447, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317)
2024-11-22T17:33:39,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64835e7e50298446 with lease ID 0xe5743446d8887f08: from storage DS-0c01abdd-98ed-4b12-b051-f60b47a52375 node DatanodeRegistration(127.0.0.1:40207, datanodeUuid=b40d762e-a70a-4faa-95f1-2220c06987cb, infoPort=44459, infoSecurePort=0, ipcPort=43447, storageInfo=lv=-57;cid=testClusterID;nsid=1009893875;c=1732296816317), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:33:39,999 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538
2024-11-22T17:33:40,071 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/zookeeper_0, clientPort=58234, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:33:40,081 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58234
2024-11-22T17:33:40,090 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:40,093 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:40,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:33:40,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:33:40,700 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b with version=8
2024-11-22T17:33:40,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:33:40,779 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-22T17:33:41,020 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:33:41,029 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,030 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,034 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:33:41,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,034 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:33:41,157 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:33:41,209 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-22T17:33:41,220 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-22T17:33:41,224 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:33:41,251 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97821 (auto-detected)
2024-11-22T17:33:41,252 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-22T17:33:41,269 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46541
2024-11-22T17:33:41,290 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46541 connecting to ZooKeeper ensemble=127.0.0.1:58234
2024-11-22T17:33:41,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:465410x0, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:33:41,419 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46541-0x101639bc0fb0000 connected
2024-11-22T17:33:41,516 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:41,520 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:41,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:33:41,535 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b, hbase.cluster.distributed=false
2024-11-22T17:33:41,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:33:41,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:33:41,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46541
2024-11-22T17:33:41,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46541
2024-11-22T17:33:41,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:33:41,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:33:41,656 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:33:41,658 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,658 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,658 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:33:41,658 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:33:41,659 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:33:41,661 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:33:41,663 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:33:41,664 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33417
2024-11-22T17:33:41,666 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33417 connecting to ZooKeeper ensemble=127.0.0.1:58234
2024-11-22T17:33:41,667 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:41,671 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:41,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334170x0, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:33:41,690 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334170x0, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:33:41,690 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33417-0x101639bc0fb0001 connected
2024-11-22T17:33:41,694 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:33:41,700 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:33:41,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:33:41,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:33:41,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33417
2024-11-22T17:33:41,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33417
2024-11-22T17:33:41,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33417
2024-11-22T17:33:41,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33417
2024-11-22T17:33:41,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33417
2024-11-22T17:33:41,724 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:46541
2024-11-22T17:33:41,725 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,46541,1732296820867
2024-11-22T17:33:41,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:33:41,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:33:41,849 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,46541,1732296820867
2024-11-22T17:33:41,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:41,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:33:41,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:41,882 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:33:41,883 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,46541,1732296820867 from backup master directory
2024-11-22T17:33:41,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,46541,1732296820867
2024-11-22T17:33:41,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:33:41,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:33:41,891 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:33:41,892 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,46541,1732296820867
2024-11-22T17:33:41,894 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-22T17:33:41,896 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-22T17:33:41,951 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase.id] with ID: 63fa0ea5-4ad9-4b04-8dd7-4deda70d9c2b
2024-11-22T17:33:41,951 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/.tmp/hbase.id
2024-11-22T17:33:41,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:33:41,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:33:41,964 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/.tmp/hbase.id]:[hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase.id]
2024-11-22T17:33:42,007 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:42,012 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:33:42,030 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-22T17:33:42,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:33:42,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:33:42,071 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:33:42,073 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:33:42,078 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:33:42,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:33:42,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:33:42,122 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store
2024-11-22T17:33:42,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:33:42,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:33:42,146 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-22T17:33:42,150 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:42,151 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:33:42,151 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:33:42,151 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:33:42,153 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:33:42,153 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:33:42,153 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:33:42,154 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296822151Disabling compacts and flushes for region at 1732296822151Disabling writes for close at 1732296822153 (+2 ms)Writing region close event to WAL at 1732296822153Closed at 1732296822153
2024-11-22T17:33:42,156 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/.initializing
2024-11-22T17:33:42,156 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/WALs/e61df7af61d2,46541,1732296820867
2024-11-22T17:33:42,177 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C46541%2C1732296820867, suffix=, logDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/WALs/e61df7af61d2,46541,1732296820867, archiveDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/oldWALs, maxLogs=10
2024-11-22T17:33:42,185 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C46541%2C1732296820867.1732296822181
2024-11-22T17:33:42,203 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/WALs/e61df7af61d2,46541,1732296820867/e61df7af61d2%2C46541%2C1732296820867.1732296822181
2024-11-22T17:33:42,211 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:33:42,212 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:33:42,212 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:42,215 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,216 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,248 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:33:42,270 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:33:42,273 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,275 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:42,276 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:33:42,278 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:33:42,279 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,280 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:33:42,280 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:33:42,282 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:33:42,283 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,283 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:33:42,284 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:33:42,286 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:33:42,286 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,287 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:33:42,288 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,292 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,293 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,298 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,299 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,302 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:33:42,308 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:33:42,312 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:33:42,314 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868891, jitterRate=0.1048528403043747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:33:42,322 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732296822227Initializing all the Stores at 1732296822230 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296822230Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296822231 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296822231Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296822231Cleaning up temporary data from old regions at 1732296822299 (+68 ms)Region opened successfully at 1732296822322 (+23 ms)
2024-11-22T17:33:42,323 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:33:42,355 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f94f83c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:33:42,383 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:33:42,393 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:33:42,393 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:33:42,395 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:33:42,397 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-22T17:33:42,402 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-22T17:33:42,402 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:33:42,428 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:33:42,437 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:33:42,489 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:33:42,492 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:33:42,494 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:33:42,501 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:33:42,504 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:33:42,508 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:33:42,520 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:33:42,522 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:33:42,533 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:33:42,552 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:33:42,564 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:33:42,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:33:42,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:33:42,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,579 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,46541,1732296820867, sessionid=0x101639bc0fb0000, setting cluster-up flag (Was=false)
2024-11-22T17:33:42,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,638 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:33:42,642 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,46541,1732296820867
2024-11-22T17:33:42,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:42,702 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:33:42,705 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,46541,1732296820867
2024-11-22T17:33:42,713 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:33:42,778 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:33:42,787 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:33:42,793 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:33:42,798 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,46541,1732296820867
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:33:42,804 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:33:42,805 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:42,806 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732296852806
2024-11-22T17:33:42,808 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:33:42,809 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:33:42,811 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:33:42,811 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:33:42,812 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:33:42,812 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:33:42,813 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:33:42,813 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:33:42,814 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(746): ClusterId : 63fa0ea5-4ad9-4b04-8dd7-4deda70d9c2b
2024-11-22T17:33:42,814 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:42,817 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:33:42,818 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:33:42,819 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,819 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:33:42,820 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:33:42,820 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:33:42,822 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:33:42,823 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:33:42,825 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296822824,5,FailOnTimeoutGroup]
2024-11-22T17:33:42,827 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296822825,5,FailOnTimeoutGroup]
2024-11-22T17:33:42,828 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:42,828 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:33:42,829 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:42,830 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:42,830 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:33:42,830 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:33:42,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:33:42,837 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:33:42,839 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:33:42,839 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b
2024-11-22T17:33:42,840 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:33:42,840 DEBUG [RS:0;e61df7af61d2:33417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7d3ef4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:33:42,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:33:42,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:33:42,857 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:33417
2024-11-22T17:33:42,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:42,859 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:33:42,861 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:33:42,861 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:33:42,861 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:33:42,862 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:33:42,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,863 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:42,864 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:33:42,864 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,46541,1732296820867 with port=33417, startcode=1732296821625
2024-11-22T17:33:42,866 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:33:42,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,868 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:42,868 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:33:42,871 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:33:42,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,873 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:42,873 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:33:42,875 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:33:42,875 DEBUG [RS:0;e61df7af61d2:33417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:33:42,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:42,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:42,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:33:42,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740
2024-11-22T17:33:42,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740
2024-11-22T17:33:42,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:33:42,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:33:42,883 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:33:42,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:33:42,890 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:33:42,891 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842851, jitterRate=0.07174086570739746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:33:42,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732296822857Initializing all the Stores at 1732296822859 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296822859Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296822859Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296822859Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296822859Cleaning up temporary data from old regions at 1732296822883 (+24 ms)Region opened successfully at 1732296822896 (+13 ms)
2024-11-22T17:33:42,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:33:42,896 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:33:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:33:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:33:42,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:33:42,898 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:33:42,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296822896Disabling compacts and flushes for region at 1732296822896Disabling writes for close at 1732296822897 (+1 ms)Writing region close event to WAL at 1732296822898 (+1 ms)Closed at 1732296822898
2024-11-22T17:33:42,902 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:33:42,902 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:33:42,909 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:33:42,919 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:33:42,922 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:33:42,937 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:33:42,943 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46541 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,33417,1732296821625
2024-11-22T17:33:42,945 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46541 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:42,958 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b
2024-11-22T17:33:42,958 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41389
2024-11-22T17:33:42,958 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:33:42,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:33:42,965 DEBUG [RS:0;e61df7af61d2:33417 {}] zookeeper.ZKUtil(111): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,33417,1732296821625
2024-11-22T17:33:42,965 WARN  [RS:0;e61df7af61d2:33417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:33:42,966 INFO  [RS:0;e61df7af61d2:33417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:33:42,966 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625
2024-11-22T17:33:42,968 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,33417,1732296821625]
2024-11-22T17:33:42,992 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:33:43,006 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:33:43,010 INFO  [RS:0;e61df7af61d2:33417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:33:43,010 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,011 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:33:43,016 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:33:43,018 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,018 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,018 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,018 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,018 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,019 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:33:43,020 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:33:43,020 DEBUG [RS:0;e61df7af61d2:33417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,021 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33417,1732296821625-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:33:43,038 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:33:43,039 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33417,1732296821625-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,039 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,040 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.Replication(171): e61df7af61d2,33417,1732296821625 started
2024-11-22T17:33:43,057 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,057 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,33417,1732296821625, RpcServer on e61df7af61d2/172.17.0.2:33417, sessionid=0x101639bc0fb0001
2024-11-22T17:33:43,058 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:33:43,058 DEBUG [RS:0;e61df7af61d2:33417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,33417,1732296821625
2024-11-22T17:33:43,058 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,33417,1732296821625'
2024-11-22T17:33:43,058 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:33:43,059 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:33:43,060 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:33:43,060 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:33:43,060 DEBUG [RS:0;e61df7af61d2:33417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,33417,1732296821625
2024-11-22T17:33:43,060 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,33417,1732296821625'
2024-11-22T17:33:43,060 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:33:43,061 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:33:43,062 DEBUG [RS:0;e61df7af61d2:33417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:33:43,062 INFO  [RS:0;e61df7af61d2:33417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:33:43,062 INFO  [RS:0;e61df7af61d2:33417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:33:43,073 WARN  [e61df7af61d2:46541 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:33:43,174 INFO  [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C33417%2C1732296821625, suffix=, logDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625, archiveDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs, maxLogs=32
2024-11-22T17:33:43,177 INFO  [RS:0;e61df7af61d2:33417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296823177
2024-11-22T17:33:43,188 INFO  [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296823177
2024-11-22T17:33:43,190 DEBUG [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459),(127.0.0.1/127.0.0.1:38499:38499)]
2024-11-22T17:33:43,326 DEBUG [e61df7af61d2:46541 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:33:43,339 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:43,345 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,33417,1732296821625, state=OPENING
2024-11-22T17:33:43,354 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:33:43,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:43,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:33:43,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:33:43,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:33:43,368 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:33:43,370 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,33417,1732296821625}]
2024-11-22T17:33:43,547 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:33:43,550 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:33:43,562 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:33:43,563 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:33:43,567 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C33417%2C1732296821625.meta, suffix=.meta, logDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625, archiveDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs, maxLogs=32
2024-11-22T17:33:43,569 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.meta.1732296823569.meta
2024-11-22T17:33:43,577 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.meta.1732296823569.meta
2024-11-22T17:33:43,579 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:33:43,583 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:33:43,584 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:33:43,586 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:33:43,590 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:33:43,594 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:33:43,594 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:43,595 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:33:43,595 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:33:43,598 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:33:43,600 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:33:43,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:43,601 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:43,601 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:33:43,603 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:33:43,603 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:43,604 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:43,604 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:33:43,606 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:33:43,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:43,606 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:43,607 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:33:43,608 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:33:43,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:43,609 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:33:43,610 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:33:43,611 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740
2024-11-22T17:33:43,614 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740
2024-11-22T17:33:43,616 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:33:43,616 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:33:43,617 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:33:43,620 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:33:43,622 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739196, jitterRate=-0.06006459891796112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:33:43,622 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:33:43,624 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732296823595Writing region info on filesystem at 1732296823595Initializing all the Stores at 1732296823597 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296823597Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296823597Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296823598 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296823598Cleaning up temporary data from old regions at 1732296823616 (+18 ms)Running coprocessor post-open hooks at 1732296823622 (+6 ms)Region opened successfully at 1732296823623 (+1 ms)
2024-11-22T17:33:43,631 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732296823539
2024-11-22T17:33:43,643 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:33:43,644 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:33:43,646 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:43,649 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,33417,1732296821625, state=OPEN
2024-11-22T17:33:43,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:33:43,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:33:43,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:33:43,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:33:43,739 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:43,745 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:33:43,745 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,33417,1732296821625 in 369 msec
2024-11-22T17:33:43,752 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:33:43,753 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 838 msec
2024-11-22T17:33:43,754 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:33:43,754 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:33:43,771 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:33:43,772 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,33417,1732296821625, seqNum=-1]
2024-11-22T17:33:43,789 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:33:43,791 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:33:43,808 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0660 sec
2024-11-22T17:33:43,808 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732296823808, completionTime=-1
2024-11-22T17:33:43,811 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:33:43,811 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:33:43,835 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:33:43,835 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732296883835
2024-11-22T17:33:43,836 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732296943835
2024-11-22T17:33:43,836 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec
2024-11-22T17:33:43,840 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,840 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,840 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,842 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:46541, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,843 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,843 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,849 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:33:43,869 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.977sec
2024-11-22T17:33:43,871 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:33:43,872 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:33:43,873 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:33:43,874 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:33:43,874 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:33:43,875 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:33:43,875 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:33:43,884 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:33:43,885 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:33:43,885 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296820867-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:33:43,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a4c629c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:33:43,948 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-22T17:33:43,949 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-22T17:33:43,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,46541,-1 for getting cluster id
2024-11-22T17:33:43,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:33:43,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '63fa0ea5-4ad9-4b04-8dd7-4deda70d9c2b'
2024-11-22T17:33:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:33:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "63fa0ea5-4ad9-4b04-8dd7-4deda70d9c2b"

2024-11-22T17:33:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29842d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:33:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,46541,-1]
2024-11-22T17:33:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:33:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:33:43,984 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:33:43,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5286c427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:33:43,988 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:33:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,33417,1732296821625, seqNum=-1]
2024-11-22T17:33:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:33:44,002 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:33:44,023 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,46541,1732296820867
2024-11-22T17:33:44,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:33:44,031 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:33:44,035 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T17:33:44,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,46541,1732296820867
2024-11-22T17:33:44,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bfd4db8
2024-11-22T17:33:44,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T17:33:44,046 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T17:33:44,049 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T17:33:44,049 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T17:33:44,054 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:33:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-22T17:33:44,064 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T17:33:44,066 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-22T17:33:44,067 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:44,069 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T17:33:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:33:44,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741835_1011 (size=389)
2024-11-22T17:33:44,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741835_1011 (size=389)
2024-11-22T17:33:44,120 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2542ee36d3079d6442cbd4440974b2c7, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b
2024-11-22T17:33:44,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741836_1012 (size=72)
2024-11-22T17:33:44,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741836_1012 (size=72)
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 2542ee36d3079d6442cbd4440974b2c7, disabling compactions & flushes
2024-11-22T17:33:44,133 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7. after waiting 0 ms
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,133 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,133 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2542ee36d3079d6442cbd4440974b2c7:
Waiting for close lock at 1732296824133Disabling compacts and flushes for region at 1732296824133Disabling writes for close at 1732296824133Writing region close event to WAL at 1732296824133Closed at 1732296824133
2024-11-22T17:33:44,135 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T17:33:44,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732296824135"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732296824135"}]},"ts":"1732296824135"}
2024-11-22T17:33:44,144 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T17:33:44,146 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T17:33:44,149 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296824146"}]},"ts":"1732296824146"}
2024-11-22T17:33:44,153 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-22T17:33:44,155 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=2542ee36d3079d6442cbd4440974b2c7, ASSIGN}]
2024-11-22T17:33:44,157 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=2542ee36d3079d6442cbd4440974b2c7, ASSIGN
2024-11-22T17:33:44,159 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=2542ee36d3079d6442cbd4440974b2c7, ASSIGN; state=OFFLINE, location=e61df7af61d2,33417,1732296821625; forceNewPlan=false, retain=false
2024-11-22T17:33:44,313 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2542ee36d3079d6442cbd4440974b2c7, regionState=OPENING, regionLocation=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:44,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=2542ee36d3079d6442cbd4440974b2c7, ASSIGN because future has completed
2024-11-22T17:33:44,324 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2542ee36d3079d6442cbd4440974b2c7, server=e61df7af61d2,33417,1732296821625}]
2024-11-22T17:33:44,485 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,485 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2542ee36d3079d6442cbd4440974b2c7, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:33:44,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:33:44,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,489 INFO  [StoreOpener-2542ee36d3079d6442cbd4440974b2c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2542ee36d3079d6442cbd4440974b2c7 
2024-11-22T17:33:44,492 INFO  [StoreOpener-2542ee36d3079d6442cbd4440974b2c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2542ee36d3079d6442cbd4440974b2c7 columnFamilyName info
2024-11-22T17:33:44,493 DEBUG [StoreOpener-2542ee36d3079d6442cbd4440974b2c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:33:44,494 INFO  [StoreOpener-2542ee36d3079d6442cbd4440974b2c7-1 {}] regionserver.HStore(327): Store=2542ee36d3079d6442cbd4440974b2c7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:33:44,494 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,496 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,497 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,500 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,504 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:33:44,504 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2542ee36d3079d6442cbd4440974b2c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=720763, jitterRate=-0.08350257575511932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:33:44,505 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:33:44,506 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2542ee36d3079d6442cbd4440974b2c7:
Running coprocessor pre-open hook at 1732296824487Writing region info on filesystem at 1732296824487Initializing all the Stores at 1732296824489 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296824489Cleaning up temporary data from old regions at 1732296824498 (+9 ms)Running coprocessor post-open hooks at 1732296824505 (+7 ms)Region opened successfully at 1732296824505
2024-11-22T17:33:44,507 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7., pid=6, masterSystemTime=1732296824478
2024-11-22T17:33:44,512 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,512 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:44,512 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2542ee36d3079d6442cbd4440974b2c7, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,33417,1732296821625
2024-11-22T17:33:44,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2542ee36d3079d6442cbd4440974b2c7, server=e61df7af61d2,33417,1732296821625 because future has completed
2024-11-22T17:33:44,523 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T17:33:44,523 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2542ee36d3079d6442cbd4440974b2c7, server=e61df7af61d2,33417,1732296821625 in 194 msec
2024-11-22T17:33:44,527 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T17:33:44,527 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=2542ee36d3079d6442cbd4440974b2c7, ASSIGN in 369 msec
2024-11-22T17:33:44,528 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T17:33:44,529 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296824528"}]},"ts":"1732296824528"}
2024-11-22T17:33:44,532 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-22T17:33:44,533 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T17:33:44,537 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 477 msec
2024-11-22T17:33:49,107 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-22T17:33:49,172 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T17:33:49,174 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-22T17:33:51,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:33:51,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T17:33:51,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-22T17:33:51,208 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T17:33:51,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:33:51,209 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T17:33:51,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:33:51,209 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T17:33:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46541 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:33:54,096 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-22T17:33:54,099 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-22T17:33:54,107 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-22T17:33:54,108 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:33:54,110 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296834109
2024-11-22T17:33:54,121 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:33:54,122 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:33:54,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:33:54,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:33:54,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:33:54,123 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296823177 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296834109
2024-11-22T17:33:54,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741833_1009 (size=451)
2024-11-22T17:33:54,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741833_1009 (size=451)
2024-11-22T17:33:54,132 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:33:54,137 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296823177 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296823177
2024-11-22T17:33:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7., hostname=e61df7af61d2,33417,1732296821625, seqNum=2]
2024-11-22T17:34:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:34:06,196 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2542ee36d3079d6442cbd4440974b2c7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:34:06,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/8caef9da6d41456ca012d4ccc70defa5 is 1080, key is row0001/info:/1732296834152/Put/seqid=0
2024-11-22T17:34:06,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741838_1014 (size=12509)
2024-11-22T17:34:06,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741838_1014 (size=12509)
2024-11-22T17:34:06,286 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/8caef9da6d41456ca012d4ccc70defa5
2024-11-22T17:34:06,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/8caef9da6d41456ca012d4ccc70defa5 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5
2024-11-22T17:34:06,361 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5, entries=7, sequenceid=11, filesize=12.2 K
2024-11-22T17:34:06,371 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 174ms, sequenceid=11, compaction requested=false
2024-11-22T17:34:06,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2542ee36d3079d6442cbd4440974b2c7:

2024-11-22T17:34:09,996 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:34:14,205 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296854205
2024-11-22T17:34:14,417 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 209 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:14,418 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:14,418 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:14,418 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:14,418 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:14,418 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:14,419 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296834109 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296854205
2024-11-22T17:34:14,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741837_1013 (size=12399)
2024-11-22T17:34:14,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741837_1013 (size=12399)
2024-11-22T17:34:14,429 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:34:14,632 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:16,836 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:19,040 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:21,244 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:34:21,244 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2542ee36d3079d6442cbd4440974b2c7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:34:21,446 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:21,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/0fd2118cc74b4005a42508fc868fc361 is 1080, key is row0008/info:/1732296848194/Put/seqid=0
2024-11-22T17:34:21,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741840_1016 (size=12509)
2024-11-22T17:34:21,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741840_1016 (size=12509)
2024-11-22T17:34:21,461 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/0fd2118cc74b4005a42508fc868fc361
2024-11-22T17:34:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/0fd2118cc74b4005a42508fc868fc361 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361
2024-11-22T17:34:21,485 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361, entries=7, sequenceid=21, filesize=12.2 K
2024-11-22T17:34:21,687 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:21,687 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 443ms, sequenceid=21, compaction requested=false
2024-11-22T17:34:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2542ee36d3079d6442cbd4440974b2c7:

2024-11-22T17:34:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-22T17:34:21,687 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:34:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5 because midkey is the same as first or last row
2024-11-22T17:34:23,448 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:23,889 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T17:34:23,890 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T17:34:25,653 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:25,655 WARN  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:25,657 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C33417%2C1732296821625:(num 1732296854205) roll requested
2024-11-22T17:34:25,657 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296865657
2024-11-22T17:34:25,865 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:25,866 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:25,866 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:25,866 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:25,866 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:25,866 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:25,867 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296854205 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296865657
2024-11-22T17:34:25,868 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:34:25,868 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296854205 is not closed yet, will try archiving it next time
2024-11-22T17:34:25,868 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296834109 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296834109
2024-11-22T17:34:25,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741839_1015 (size=7739)
2024-11-22T17:34:25,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741839_1015 (size=7739)
2024-11-22T17:34:27,861 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:29,487 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 2542ee36d3079d6442cbd4440974b2c7, had cached 0 bytes from a total of 25018
2024-11-22T17:34:30,067 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:32,271 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:34,477 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:36,481 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T17:34:36,482 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296876481
2024-11-22T17:34:39,996 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:34:41,493 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:41,496 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:41,497 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C33417%2C1732296821625:(num 1732296876481) roll requested
2024-11-22T17:34:41,497 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:41,497 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:41,498 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:41,498 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:41,498 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:41,499 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296865657 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296876481
2024-11-22T17:34:41,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741841_1017 (size=4753)
2024-11-22T17:34:41,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741841_1017 (size=4753)
2024-11-22T17:34:41,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459),(127.0.0.1/127.0.0.1:38499:38499)]
2024-11-22T17:34:41,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296865657 is not closed yet, will try archiving it next time
2024-11-22T17:34:41,508 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296881508
2024-11-22T17:34:46,511 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:46,511 WARN  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:34:46,512 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2542ee36d3079d6442cbd4440974b2c7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:34:46,516 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:46,516 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:48,512 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T17:34:51,515 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:51,515 WARN  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK], DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK]]
2024-11-22T17:34:51,515 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:51,516 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:51,516 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:51,517 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:51,517 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:51,518 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296876481 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508
2024-11-22T17:34:51,520 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:34:51,521 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296876481 is not closed yet, will try archiving it next time
2024-11-22T17:34:51,521 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C33417%2C1732296821625:(num 1732296881508) roll requested
2024-11-22T17:34:51,522 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296891521
2024-11-22T17:34:51,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741842_1018 (size=1569)
2024-11-22T17:34:51,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741842_1018 (size=1569)
2024-11-22T17:34:51,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/cc46aa487eb344c6bc4d0341e36f758c is 1080, key is row0015/info:/1732296863247/Put/seqid=0
2024-11-22T17:34:51,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741844_1020 (size=12509)
2024-11-22T17:34:51,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741844_1020 (size=12509)
2024-11-22T17:34:51,936 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/cc46aa487eb344c6bc4d0341e36f758c
2024-11-22T17:34:51,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/cc46aa487eb344c6bc4d0341e36f758c as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c
2024-11-22T17:34:51,961 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c, entries=7, sequenceid=31, filesize=12.2 K
2024-11-22T17:34:56,530 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:56,531 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:56,962 INFO  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:56,962 WARN  [FSHLog-0-hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b-prefix:e61df7af61d2,33417,1732296821625 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33349,DS-a8b3852b-ce26-4c79-be4a-d60da984df2f,DISK], DatanodeInfoWithStorage[127.0.0.1:40207,DS-b07b3be2-f6d8-47ac-8f26-34457bdea851,DISK]]
2024-11-22T17:34:56,962 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 10451ms, sequenceid=31, compaction requested=true
2024-11-22T17:34:56,962 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2542ee36d3079d6442cbd4440974b2c7:

2024-11-22T17:34:56,963 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,963 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-22T17:34:56,963 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:34:56,963 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,963 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5 because midkey is the same as first or last row
2024-11-22T17:34:56,963 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,963 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,963 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296891521
2024-11-22T17:34:56,964 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459),(127.0.0.1/127.0.0.1:38499:38499)]
2024-11-22T17:34:56,964 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 is not closed yet, will try archiving it next time
2024-11-22T17:34:56,964 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296854205 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296854205
2024-11-22T17:34:56,964 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C33417%2C1732296821625:(num 1732296896964) roll requested
2024-11-22T17:34:56,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2542ee36d3079d6442cbd4440974b2c7:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:34:56,964 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296896964
2024-11-22T17:34:56,966 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296865657 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296865657
2024-11-22T17:34:56,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:34:56,967 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:34:56,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741843_1019 (size=438)
2024-11-22T17:34:56,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741843_1019 (size=438)
2024-11-22T17:34:56,968 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296876481 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296876481
2024-11-22T17:34:56,969 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:34:56,971 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HStore(1541): 2542ee36d3079d6442cbd4440974b2c7/info is initiating minor compaction (all files)
2024-11-22T17:34:56,972 INFO  [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2542ee36d3079d6442cbd4440974b2c7/info in TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:34:56,972 INFO  [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c] into tmpdir=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp, totalSize=36.6 K
2024-11-22T17:34:56,973 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,974 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,974 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8caef9da6d41456ca012d4ccc70defa5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732296834152
2024-11-22T17:34:56,974 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,974 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,974 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,974 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296891521 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296896964
2024-11-22T17:34:56,974 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd2118cc74b4005a42508fc868fc361, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732296848194
2024-11-22T17:34:56,975 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc46aa487eb344c6bc4d0341e36f758c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732296863247
2024-11-22T17:34:56,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741845_1021 (size=93)
2024-11-22T17:34:56,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741845_1021 (size=93)
2024-11-22T17:34:56,977 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 is not closed yet, will try archiving it next time
2024-11-22T17:34:56,977 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296891521 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296891521
2024-11-22T17:34:56,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44459:44459),(127.0.0.1/127.0.0.1:38499:38499)]
2024-11-22T17:34:56,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 is not closed yet, will try archiving it next time
2024-11-22T17:34:56,981 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33417%2C1732296821625.1732296896981
2024-11-22T17:34:56,990 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,990 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,991 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,991 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,991 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:34:56,991 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296896964 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296896981
2024-11-22T17:34:56,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741846_1022 (size=1258)
2024-11-22T17:34:56,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741846_1022 (size=1258)
2024-11-22T17:34:56,995 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38499:38499),(127.0.0.1/127.0.0.1:44459:44459)]
2024-11-22T17:34:56,995 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 is not closed yet, will try archiving it next time
2024-11-22T17:34:56,995 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296896964 is not closed yet, will try archiving it next time
2024-11-22T17:34:56,996 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 is not closed yet, will try archiving it next time
2024-11-22T17:34:57,010 INFO  [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2542ee36d3079d6442cbd4440974b2c7#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:34:57,011 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/9eae48c70a1a4ceaa7d29e5b623d9b62 is 1080, key is row0001/info:/1732296834152/Put/seqid=0
2024-11-22T17:34:57,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741848_1024 (size=27710)
2024-11-22T17:34:57,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741848_1024 (size=27710)
2024-11-22T17:34:57,028 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/9eae48c70a1a4ceaa7d29e5b623d9b62 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/9eae48c70a1a4ceaa7d29e5b623d9b62
2024-11-22T17:34:57,044 INFO  [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2542ee36d3079d6442cbd4440974b2c7/info of 2542ee36d3079d6442cbd4440974b2c7 into 9eae48c70a1a4ceaa7d29e5b623d9b62(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:34:57,044 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2542ee36d3079d6442cbd4440974b2c7:

2024-11-22T17:34:57,046 INFO  [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7., storeName=2542ee36d3079d6442cbd4440974b2c7/info, priority=13, startTime=1732296896964; duration=0sec
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/9eae48c70a1a4ceaa7d29e5b623d9b62 because midkey is the same as first or last row
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:34:57,046 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/9eae48c70a1a4ceaa7d29e5b623d9b62 because midkey is the same as first or last row
2024-11-22T17:34:57,047 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-22T17:34:57,047 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:34:57,047 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/9eae48c70a1a4ceaa7d29e5b623d9b62 because midkey is the same as first or last row
2024-11-22T17:34:57,047 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:34:57,047 DEBUG [RS:0;e61df7af61d2:33417-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2542ee36d3079d6442cbd4440974b2c7:info
2024-11-22T17:34:57,368 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/WALs/e61df7af61d2,33417,1732296821625/e61df7af61d2%2C33417%2C1732296821625.1732296881508 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs/e61df7af61d2%2C33417%2C1732296821625.1732296881508
2024-11-22T17:35:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33417 {}] regionserver.HRegion(8855): Flush requested on 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:35:09,012 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2542ee36d3079d6442cbd4440974b2c7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:35:09,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/724ec9fc6167440eb1d0bd05b120f82e is 1080, key is row0022/info:/1732296896982/Put/seqid=0
2024-11-22T17:35:09,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741849_1025 (size=12509)
2024-11-22T17:35:09,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741849_1025 (size=12509)
2024-11-22T17:35:09,033 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/724ec9fc6167440eb1d0bd05b120f82e
2024-11-22T17:35:09,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/724ec9fc6167440eb1d0bd05b120f82e as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/724ec9fc6167440eb1d0bd05b120f82e
2024-11-22T17:35:09,053 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/724ec9fc6167440eb1d0bd05b120f82e, entries=7, sequenceid=42, filesize=12.2 K
2024-11-22T17:35:09,054 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 43ms, sequenceid=42, compaction requested=false
2024-11-22T17:35:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2542ee36d3079d6442cbd4440974b2c7:

2024-11-22T17:35:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-22T17:35:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:09,054 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/9eae48c70a1a4ceaa7d29e5b623d9b62 because midkey is the same as first or last row
2024-11-22T17:35:09,997 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:35:14,488 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 2542ee36d3079d6442cbd4440974b2c7, had cached 0 bytes from a total of 40219
2024-11-22T17:35:17,029 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:35:17,030 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:35:17,030 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:17,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:17,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:17,037 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:35:17,037 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:35:17,037 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=372944084, stopped=false
2024-11-22T17:35:17,037 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,46541,1732296820867
2024-11-22T17:35:17,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:17,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:17,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:17,092 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:35:17,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:17,093 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:35:17,093 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:17,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:17,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:17,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:17,094 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,33417,1732296821625' *****
2024-11-22T17:35:17,094 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:35:17,094 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:35:17,094 INFO  [RS:0;e61df7af61d2:33417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:35:17,094 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:35:17,095 INFO  [RS:0;e61df7af61d2:33417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:35:17,095 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(3091): Received CLOSE for 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:35:17,096 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,33417,1732296821625
2024-11-22T17:35:17,096 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:35:17,096 INFO  [RS:0;e61df7af61d2:33417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:33417.
2024-11-22T17:35:17,096 DEBUG [RS:0;e61df7af61d2:33417 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:17,096 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2542ee36d3079d6442cbd4440974b2c7, disabling compactions & flushes
2024-11-22T17:35:17,096 DEBUG [RS:0;e61df7af61d2:33417 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:17,096 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:35:17,096 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:35:17,097 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7. after waiting 0 ms
2024-11-22T17:35:17,097 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:35:17,097 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:35:17,097 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:35:17,097 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:35:17,097 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2542ee36d3079d6442cbd4440974b2c7 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-22T17:35:17,097 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:35:17,100 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T17:35:17,100 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:35:17,101 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:35:17,101 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1325): Online Regions={2542ee36d3079d6442cbd4440974b2c7=TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7., 1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:35:17,101 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:35:17,101 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:35:17,101 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:35:17,101 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-22T17:35:17,102 DEBUG [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2542ee36d3079d6442cbd4440974b2c7
2024-11-22T17:35:17,105 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/187ca024fbe846e7a4dafa1a87183e27 is 1080, key is row0029/info:/1732296911014/Put/seqid=0
2024-11-22T17:35:17,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741850_1026 (size=8193)
2024-11-22T17:35:17,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741850_1026 (size=8193)
2024-11-22T17:35:17,122 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/187ca024fbe846e7a4dafa1a87183e27
2024-11-22T17:35:17,126 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/info/310cd2aa9b544729847cf66a164debb7 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7./info:regioninfo/1732296824512/Put/seqid=0
2024-11-22T17:35:17,139 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/.tmp/info/187ca024fbe846e7a4dafa1a87183e27 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/187ca024fbe846e7a4dafa1a87183e27
2024-11-22T17:35:17,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741851_1027 (size=7016)
2024-11-22T17:35:17,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741851_1027 (size=7016)
2024-11-22T17:35:17,144 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/info/310cd2aa9b544729847cf66a164debb7
2024-11-22T17:35:17,150 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/187ca024fbe846e7a4dafa1a87183e27, entries=3, sequenceid=48, filesize=8.0 K
2024-11-22T17:35:17,152 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 54ms, sequenceid=48, compaction requested=true
2024-11-22T17:35:17,152 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c] to archive
2024-11-22T17:35:17,156 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:35:17,159 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/8caef9da6d41456ca012d4ccc70defa5
2024-11-22T17:35:17,162 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361 to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/0fd2118cc74b4005a42508fc868fc361
2024-11-22T17:35:17,165 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/archive/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/info/cc46aa487eb344c6bc4d0341e36f758c
2024-11-22T17:35:17,174 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/ns/c78bfc88c75a4ba9b486b6ff291fcfbf is 43, key is default/ns:d/1732296823794/Put/seqid=0
2024-11-22T17:35:17,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741852_1028 (size=5153)
2024-11-22T17:35:17,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741852_1028 (size=5153)
2024-11-22T17:35:17,183 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/ns/c78bfc88c75a4ba9b486b6ff291fcfbf
2024-11-22T17:35:17,179 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e61df7af61d2:46541 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T17:35:17,185 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8caef9da6d41456ca012d4ccc70defa5=12509, 0fd2118cc74b4005a42508fc868fc361=12509, cc46aa487eb344c6bc4d0341e36f758c=12509]
2024-11-22T17:35:17,191 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/default/TestLogRolling-testSlowSyncLogRolling/2542ee36d3079d6442cbd4440974b2c7/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-22T17:35:17,194 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:35:17,194 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2542ee36d3079d6442cbd4440974b2c7:
Waiting for close lock at 1732296917096Running coprocessor pre-close hooks at 1732296917096Disabling compacts and flushes for region at 1732296917096Disabling writes for close at 1732296917097 (+1 ms)Obtaining lock to block concurrent updates at 1732296917097Preparing flush snapshotting stores in 2542ee36d3079d6442cbd4440974b2c7 at 1732296917097Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732296917097Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7. at 1732296917099 (+2 ms)Flushing 2542ee36d3079d6442cbd4440974b2c7/info: creating writer at 1732296917099Flushing 2542ee36d3079d6442cbd4440974b2c7/info: appending metadata at 1732296917105 (+6 ms)Flushing 2542ee36d3079d6442cbd4440974b2c7/info: closing flushed file at 1732296917105Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@435dd0ea: reopening flushed file at 1732296917134 (+29 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 2542ee36d3079d6442cbd4440974b2c7 in 54ms, sequenceid=48, compaction requested=true at 1732296917152 (+18 ms)Writing region close event to WAL at 1732296917186 (+34 ms)Running coprocessor post-close hooks at 1732296917192 (+6 ms)Closed at 1732296917194 (+2 ms)
2024-11-22T17:35:17,195 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732296824048.2542ee36d3079d6442cbd4440974b2c7.
2024-11-22T17:35:17,212 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/table/ba82645cb4f9439782ac9c7da984b6cc is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732296824528/Put/seqid=0
2024-11-22T17:35:17,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741853_1029 (size=5396)
2024-11-22T17:35:17,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741853_1029 (size=5396)
2024-11-22T17:35:17,221 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/table/ba82645cb4f9439782ac9c7da984b6cc
2024-11-22T17:35:17,232 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/info/310cd2aa9b544729847cf66a164debb7 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/info/310cd2aa9b544729847cf66a164debb7
2024-11-22T17:35:17,242 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/info/310cd2aa9b544729847cf66a164debb7, entries=10, sequenceid=11, filesize=6.9 K
2024-11-22T17:35:17,244 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/ns/c78bfc88c75a4ba9b486b6ff291fcfbf as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/ns/c78bfc88c75a4ba9b486b6ff291fcfbf
2024-11-22T17:35:17,255 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/ns/c78bfc88c75a4ba9b486b6ff291fcfbf, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T17:35:17,257 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/.tmp/table/ba82645cb4f9439782ac9c7da984b6cc as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/table/ba82645cb4f9439782ac9c7da984b6cc
2024-11-22T17:35:17,267 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/table/ba82645cb4f9439782ac9c7da984b6cc, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T17:35:17,269 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 167ms, sequenceid=11, compaction requested=false
2024-11-22T17:35:17,275 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T17:35:17,276 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:35:17,276 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:17,276 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296917100Running coprocessor pre-close hooks at 1732296917100Disabling compacts and flushes for region at 1732296917100Disabling writes for close at 1732296917101 (+1 ms)Obtaining lock to block concurrent updates at 1732296917101Preparing flush snapshotting stores in 1588230740 at 1732296917102 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732296917102Flushing stores of hbase:meta,,1.1588230740 at 1732296917103 (+1 ms)Flushing 1588230740/info: creating writer at 1732296917104 (+1 ms)Flushing 1588230740/info: appending metadata at 1732296917126 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732296917126Flushing 1588230740/ns: creating writer at 1732296917154 (+28 ms)Flushing 1588230740/ns: appending metadata at 1732296917173 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732296917173Flushing 1588230740/table: creating writer at 1732296917195 (+22 ms)Flushing 1588230740/table: appending metadata at 1732296917211 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732296917211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6210853a: reopening flushed file at 1732296917231 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b84d881: reopening flushed file at 1732296917242 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26f1ffa8: reopening flushed file at 1732296917255 (+13 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 167ms, sequenceid=11, compaction requested=false at 1732296917269 (+14 ms)Writing region close event to WAL at 1732296917270 (+1 ms)Running coprocessor post-close hooks at 1732296917276 (+6 ms)Closed at 1732296917276
2024-11-22T17:35:17,277 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:17,302 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,33417,1732296821625; all regions closed.
2024-11-22T17:35:17,304 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,304 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,304 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,304 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,304 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741834_1010 (size=3066)
2024-11-22T17:35:17,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741834_1010 (size=3066)
2024-11-22T17:35:17,312 DEBUG [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs
2024-11-22T17:35:17,312 INFO  [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C33417%2C1732296821625.meta:.meta(num 1732296823569)
2024-11-22T17:35:17,312 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,312 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,312 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,313 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,313 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741847_1023 (size=12695)
2024-11-22T17:35:17,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741847_1023 (size=12695)
2024-11-22T17:35:17,319 DEBUG [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/oldWALs
2024-11-22T17:35:17,320 INFO  [RS:0;e61df7af61d2:33417 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C33417%2C1732296821625:(num 1732296896981)
2024-11-22T17:35:17,320 DEBUG [RS:0;e61df7af61d2:33417 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:17,320 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:35:17,320 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:35:17,320 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:35:17,320 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:35:17,320 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:35:17,321 INFO  [RS:0;e61df7af61d2:33417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33417
2024-11-22T17:35:17,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,33417,1732296821625
2024-11-22T17:35:17,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:35:17,334 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:35:17,345 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,33417,1732296821625]
2024-11-22T17:35:17,355 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,33417,1732296821625 already deleted, retry=false
2024-11-22T17:35:17,355 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,33417,1732296821625 expired; onlineServers=0
2024-11-22T17:35:17,356 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,46541,1732296820867' *****
2024-11-22T17:35:17,356 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:35:17,356 INFO  [M:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:35:17,356 INFO  [M:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:35:17,356 DEBUG [M:0;e61df7af61d2:46541 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:35:17,356 DEBUG [M:0;e61df7af61d2:46541 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:35:17,356 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:35:17,356 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296822824 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296822824,5,FailOnTimeoutGroup]
2024-11-22T17:35:17,356 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296822825 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296822825,5,FailOnTimeoutGroup]
2024-11-22T17:35:17,357 INFO  [M:0;e61df7af61d2:46541 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:35:17,357 INFO  [M:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:35:17,357 DEBUG [M:0;e61df7af61d2:46541 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:35:17,357 INFO  [M:0;e61df7af61d2:46541 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:35:17,357 INFO  [M:0;e61df7af61d2:46541 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:35:17,358 INFO  [M:0;e61df7af61d2:46541 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:35:17,358 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:35:17,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:35:17,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:17,366 DEBUG [M:0;e61df7af61d2:46541 {}] zookeeper.ZKUtil(347): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:35:17,366 WARN  [M:0;e61df7af61d2:46541 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:35:17,367 INFO  [M:0;e61df7af61d2:46541 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/.lastflushedseqids
2024-11-22T17:35:17,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741854_1030 (size=130)
2024-11-22T17:35:17,384 INFO  [M:0;e61df7af61d2:46541 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:35:17,384 INFO  [M:0;e61df7af61d2:46541 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:35:17,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741854_1030 (size=130)
2024-11-22T17:35:17,385 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:35:17,385 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:17,385 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:17,385 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:35:17,385 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:17,385 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-11-22T17:35:17,404 DEBUG [M:0;e61df7af61d2:46541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6957bbf32cfb4b5eb63e3521f3d9ee47 is 82, key is hbase:meta,,1/info:regioninfo/1732296823645/Put/seqid=0
2024-11-22T17:35:17,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741855_1031 (size=5672)
2024-11-22T17:35:17,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741855_1031 (size=5672)
2024-11-22T17:35:17,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:17,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33417-0x101639bc0fb0001, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:17,446 INFO  [RS:0;e61df7af61d2:33417 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:35:17,446 INFO  [RS:0;e61df7af61d2:33417 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,33417,1732296821625; zookeeper connection closed.
2024-11-22T17:35:17,446 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d8816e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d8816e
2024-11-22T17:35:17,447 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:35:17,811 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6957bbf32cfb4b5eb63e3521f3d9ee47
2024-11-22T17:35:17,845 DEBUG [M:0;e61df7af61d2:46541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d589d65932b34a11906a3d8e8422bb19 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732296824536/Put/seqid=0
2024-11-22T17:35:17,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741856_1032 (size=6247)
2024-11-22T17:35:17,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741856_1032 (size=6247)
2024-11-22T17:35:17,852 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d589d65932b34a11906a3d8e8422bb19
2024-11-22T17:35:17,859 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d589d65932b34a11906a3d8e8422bb19
2024-11-22T17:35:17,874 DEBUG [M:0;e61df7af61d2:46541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f55249d51a02462983de5c14943e27e9 is 69, key is e61df7af61d2,33417,1732296821625/rs:state/1732296822947/Put/seqid=0
2024-11-22T17:35:17,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741857_1033 (size=5156)
2024-11-22T17:35:17,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741857_1033 (size=5156)
2024-11-22T17:35:17,881 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f55249d51a02462983de5c14943e27e9
2024-11-22T17:35:17,905 DEBUG [M:0;e61df7af61d2:46541 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be68e68900b942c99d826d117b62c0bf is 52, key is load_balancer_on/state:d/1732296824028/Put/seqid=0
2024-11-22T17:35:17,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741858_1034 (size=5056)
2024-11-22T17:35:17,911 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741858_1034 (size=5056)
2024-11-22T17:35:17,912 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be68e68900b942c99d826d117b62c0bf
2024-11-22T17:35:17,921 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6957bbf32cfb4b5eb63e3521f3d9ee47 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6957bbf32cfb4b5eb63e3521f3d9ee47
2024-11-22T17:35:17,929 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6957bbf32cfb4b5eb63e3521f3d9ee47, entries=8, sequenceid=59, filesize=5.5 K
2024-11-22T17:35:17,931 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d589d65932b34a11906a3d8e8422bb19 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d589d65932b34a11906a3d8e8422bb19
2024-11-22T17:35:17,939 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d589d65932b34a11906a3d8e8422bb19
2024-11-22T17:35:17,939 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d589d65932b34a11906a3d8e8422bb19, entries=6, sequenceid=59, filesize=6.1 K
2024-11-22T17:35:17,941 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f55249d51a02462983de5c14943e27e9 as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f55249d51a02462983de5c14943e27e9
2024-11-22T17:35:17,949 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f55249d51a02462983de5c14943e27e9, entries=1, sequenceid=59, filesize=5.0 K
2024-11-22T17:35:17,950 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/be68e68900b942c99d826d117b62c0bf as hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/be68e68900b942c99d826d117b62c0bf
2024-11-22T17:35:17,958 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/be68e68900b942c99d826d117b62c0bf, entries=1, sequenceid=59, filesize=4.9 K
2024-11-22T17:35:17,960 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 575ms, sequenceid=59, compaction requested=false
2024-11-22T17:35:17,961 INFO  [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:17,962 DEBUG [M:0;e61df7af61d2:46541 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296917385Disabling compacts and flushes for region at 1732296917385Disabling writes for close at 1732296917385Obtaining lock to block concurrent updates at 1732296917385Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732296917385Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1732296917386 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732296917387 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732296917387Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732296917403 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732296917403Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732296917824 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732296917844 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732296917844Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732296917859 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732296917873 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732296917873Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732296917888 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732296917905 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732296917905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52e6c6d7: reopening flushed file at 1732296917919 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3269e99: reopening flushed file at 1732296917929 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cab57fb: reopening flushed file at 1732296917939 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@516b746c: reopening flushed file at 1732296917949 (+10 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 575ms, sequenceid=59, compaction requested=false at 1732296917960 (+11 ms)Writing region close event to WAL at 1732296917961 (+1 ms)Closed at 1732296917961
2024-11-22T17:35:17,962 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,963 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,963 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,963 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,963 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:17,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40207 is added to blk_1073741830_1006 (size=27973)
2024-11-22T17:35:17,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33349 is added to blk_1073741830_1006 (size=27973)
2024-11-22T17:35:17,966 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:35:17,966 INFO  [M:0;e61df7af61d2:46541 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:35:17,966 INFO  [M:0;e61df7af61d2:46541 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46541
2024-11-22T17:35:17,966 INFO  [M:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:35:18,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:18,119 INFO  [M:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:35:18,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46541-0x101639bc0fb0000, quorum=127.0.0.1:58234, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:18,127 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c963ecd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:18,129 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:18,129 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:18,130 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:18,130 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:18,133 WARN  [BP-200041043-172.17.0.2-1732296816317 heartbeating to localhost/127.0.0.1:41389 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:18,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:18,133 WARN  [BP-200041043-172.17.0.2-1732296816317 heartbeating to localhost/127.0.0.1:41389 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-200041043-172.17.0.2-1732296816317 (Datanode Uuid d476f111-402c-449b-9df8-2f45e64954d8) service to localhost/127.0.0.1:41389
2024-11-22T17:35:18,133 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:18,134 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data3/current/BP-200041043-172.17.0.2-1732296816317 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:18,135 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data4/current/BP-200041043-172.17.0.2-1732296816317 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:18,135 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:18,138 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f93babe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:18,138 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:18,139 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:18,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:18,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:18,140 WARN  [BP-200041043-172.17.0.2-1732296816317 heartbeating to localhost/127.0.0.1:41389 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:18,140 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:18,140 WARN  [BP-200041043-172.17.0.2-1732296816317 heartbeating to localhost/127.0.0.1:41389 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-200041043-172.17.0.2-1732296816317 (Datanode Uuid b40d762e-a70a-4faa-95f1-2220c06987cb) service to localhost/127.0.0.1:41389
2024-11-22T17:35:18,140 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:18,141 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data1/current/BP-200041043-172.17.0.2-1732296816317 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:18,141 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/cluster_8cdc0c96-2b77-9668-dd92-4bd586c0cbf0/data/data2/current/BP-200041043-172.17.0.2-1732296816317 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:18,142 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:18,150 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de997b9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:35:18,150 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:18,150 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:18,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:18,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:18,159 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:35:18,190 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:35:18,199 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41389
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41389 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41389
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/e61df7af61d2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:41389
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41389
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/e61df7af61d2:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/e61df7af61d2:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@45316768
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41389 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/e61df7af61d2:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41389
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:41389 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=225 (was 376), ProcessCount=11 (was 11), AvailableMemoryMB=2845 (was 3109)
2024-11-22T17:35:18,205 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=225, ProcessCount=11, AvailableMemoryMB=2844
2024-11-22T17:35:18,205 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.log.dir so I do NOT create it in target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ce0850dd-3406-bbe5-e33c-f56f392e6538/hadoop.tmp.dir so I do NOT create it in target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac, deleteOnExit=true
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/test.cache.data in system properties and HBase conf
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:35:18,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:35:18,206 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:35:18,207 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:35:18,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:35:18,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:35:18,208 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:35:18,223 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:35:18,781 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:18,787 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:18,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:18,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:18,788 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:35:18,789 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:18,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:18,790 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:18,884 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2787d4b6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/java.io.tmpdir/jetty-localhost-42023-hadoop-hdfs-3_4_1-tests_jar-_-any-6381633346536014746/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:35:18,885 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:42023}
2024-11-22T17:35:18,885 INFO  [Time-limited test {}] server.Server(415): Started @104923ms
2024-11-22T17:35:18,897 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:35:19,026 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:35:19,188 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:19,192 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:19,193 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:19,193 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:19,193 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:35:19,194 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a3a779{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:19,194 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4bedb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:19,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a843c6e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/java.io.tmpdir/jetty-localhost-40209-hadoop-hdfs-3_4_1-tests_jar-_-any-10489124234438911328/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:19,288 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17783943{HTTP/1.1, (http/1.1)}{localhost:40209}
2024-11-22T17:35:19,288 INFO  [Time-limited test {}] server.Server(415): Started @105326ms
2024-11-22T17:35:19,290 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:19,322 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:19,326 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:19,327 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:19,327 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:19,327 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:35:19,327 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c5ed954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:19,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1db3e806{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:19,425 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@513d1177{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/java.io.tmpdir/jetty-localhost-44777-hadoop-hdfs-3_4_1-tests_jar-_-any-11038183700451834024/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:19,426 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c0a2992{HTTP/1.1, (http/1.1)}{localhost:44777}
2024-11-22T17:35:19,426 INFO  [Time-limited test {}] server.Server(415): Started @105463ms
2024-11-22T17:35:19,427 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:20,482 WARN  [Thread-451 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data1/current/BP-846298224-172.17.0.2-1732296918235/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:20,482 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data2/current/BP-846298224-172.17.0.2-1732296918235/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:20,500 WARN  [Thread-415 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:20,502 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc160449d73cb788 with lease ID 0x2a9af91f03d3e5a8: Processing first storage report for DS-2ea26aae-3633-40ae-84d7-982789689d3a from datanode DatanodeRegistration(127.0.0.1:40487, datanodeUuid=4f40fd81-c871-43ed-9746-b6f53c0601ff, infoPort=33311, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235)
2024-11-22T17:35:20,503 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc160449d73cb788 with lease ID 0x2a9af91f03d3e5a8: from storage DS-2ea26aae-3633-40ae-84d7-982789689d3a node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=4f40fd81-c871-43ed-9746-b6f53c0601ff, infoPort=33311, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:20,503 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc160449d73cb788 with lease ID 0x2a9af91f03d3e5a8: Processing first storage report for DS-56a93d33-93e8-4ea0-8cc9-49f88fc6df68 from datanode DatanodeRegistration(127.0.0.1:40487, datanodeUuid=4f40fd81-c871-43ed-9746-b6f53c0601ff, infoPort=33311, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235)
2024-11-22T17:35:20,503 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc160449d73cb788 with lease ID 0x2a9af91f03d3e5a8: from storage DS-56a93d33-93e8-4ea0-8cc9-49f88fc6df68 node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=4f40fd81-c871-43ed-9746-b6f53c0601ff, infoPort=33311, infoSecurePort=0, ipcPort=41147, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:20,632 WARN  [Thread-462 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data3/current/BP-846298224-172.17.0.2-1732296918235/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:20,632 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data4/current/BP-846298224-172.17.0.2-1732296918235/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:20,651 WARN  [Thread-438 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:20,653 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56251a34d734c925 with lease ID 0x2a9af91f03d3e5a9: Processing first storage report for DS-8200922c-09bb-47e4-a08d-6911c665aa6f from datanode DatanodeRegistration(127.0.0.1:44933, datanodeUuid=ca20f399-658e-42bc-8894-5e2cc30fc9b1, infoPort=35453, infoSecurePort=0, ipcPort=35365, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235)
2024-11-22T17:35:20,653 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56251a34d734c925 with lease ID 0x2a9af91f03d3e5a9: from storage DS-8200922c-09bb-47e4-a08d-6911c665aa6f node DatanodeRegistration(127.0.0.1:44933, datanodeUuid=ca20f399-658e-42bc-8894-5e2cc30fc9b1, infoPort=35453, infoSecurePort=0, ipcPort=35365, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:20,654 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56251a34d734c925 with lease ID 0x2a9af91f03d3e5a9: Processing first storage report for DS-806bd4bc-43bd-4a7a-83a0-7a9bc763a1c0 from datanode DatanodeRegistration(127.0.0.1:44933, datanodeUuid=ca20f399-658e-42bc-8894-5e2cc30fc9b1, infoPort=35453, infoSecurePort=0, ipcPort=35365, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235)
2024-11-22T17:35:20,654 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56251a34d734c925 with lease ID 0x2a9af91f03d3e5a9: from storage DS-806bd4bc-43bd-4a7a-83a0-7a9bc763a1c0 node DatanodeRegistration(127.0.0.1:44933, datanodeUuid=ca20f399-658e-42bc-8894-5e2cc30fc9b1, infoPort=35453, infoSecurePort=0, ipcPort=35365, storageInfo=lv=-57;cid=testClusterID;nsid=335533709;c=1732296918235), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:20,669 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05
2024-11-22T17:35:20,672 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/zookeeper_0, clientPort=50743, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:35:20,673 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50743
2024-11-22T17:35:20,673 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,675 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:35:20,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:35:20,689 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe with version=8
2024-11-22T17:35:20,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:35:20,692 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:35:20,692 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:35:20,692 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32931
2024-11-22T17:35:20,694 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32931 connecting to ZooKeeper ensemble=127.0.0.1:50743
2024-11-22T17:35:20,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:329310x0, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:35:20,748 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32931-0x101639d4a0b0000 connected
2024-11-22T17:35:20,832 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,834 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:20,838 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe, hbase.cluster.distributed=false
2024-11-22T17:35:20,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:35:20,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32931
2024-11-22T17:35:20,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32931
2024-11-22T17:35:20,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32931
2024-11-22T17:35:20,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32931
2024-11-22T17:35:20,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32931
2024-11-22T17:35:20,861 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:35:20,862 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:35:20,863 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46541
2024-11-22T17:35:20,864 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46541 connecting to ZooKeeper ensemble=127.0.0.1:50743
2024-11-22T17:35:20,865 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,867 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465410x0, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:35:20,882 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465410x0, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:20,882 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46541-0x101639d4a0b0001 connected
2024-11-22T17:35:20,882 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:35:20,883 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:35:20,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:35:20,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:35:20,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:35:20,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46541
2024-11-22T17:35:20,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46541
2024-11-22T17:35:20,893 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:35:20,893 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46541
2024-11-22T17:35:20,910 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:32931
2024-11-22T17:35:20,910 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,32931,1732296920691
2024-11-22T17:35:20,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:20,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:20,924 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,32931,1732296920691
2024-11-22T17:35:20,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:20,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:35:20,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:20,938 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:35:20,939 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,32931,1732296920691 from backup master directory
2024-11-22T17:35:20,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,32931,1732296920691
2024-11-22T17:35:20,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:20,948 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:35:20,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:20,948 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,32931,1732296920691
2024-11-22T17:35:20,957 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/hbase.id] with ID: e60fcdae-4b2f-4758-b562-a011fbcfaa93
2024-11-22T17:35:20,957 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/.tmp/hbase.id
2024-11-22T17:35:20,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:35:20,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:35:20,965 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/.tmp/hbase.id]:[hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/hbase.id]
2024-11-22T17:35:20,979 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:20,980 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:35:20,981 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T17:35:20,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:20,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:35:21,002 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:35:21,003 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:35:21,003 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:35:21,004 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:21,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:35:21,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:35:21,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:35:21,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:35:21,209 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:35:21,211 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-22T17:35:21,417 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store
2024-11-22T17:35:21,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:35:21,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:35:21,431 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:21,431 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:35:21,432 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:21,432 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:21,432 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:35:21,432 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:21,432 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:21,432 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296921431Disabling compacts and flushes for region at 1732296921431Disabling writes for close at 1732296921432 (+1 ms)Writing region close event to WAL at 1732296921432Closed at 1732296921432
2024-11-22T17:35:21,434 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/.initializing
2024-11-22T17:35:21,434 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/WALs/e61df7af61d2,32931,1732296920691
2024-11-22T17:35:21,438 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C32931%2C1732296920691, suffix=, logDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/WALs/e61df7af61d2,32931,1732296920691, archiveDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/oldWALs, maxLogs=10
2024-11-22T17:35:21,438 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C32931%2C1732296920691.1732296921438
2024-11-22T17:35:21,445 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/WALs/e61df7af61d2,32931,1732296920691/e61df7af61d2%2C32931%2C1732296920691.1732296921438
2024-11-22T17:35:21,448 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33311:33311),(127.0.0.1/127.0.0.1:35453:35453)]
2024-11-22T17:35:21,452 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:35:21,452 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:21,453 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,453 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,454 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:21,456 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:35:21,456 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,457 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:21,457 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:21,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:35:21,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:21,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:21,462 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:35:21,462 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:21,463 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:21,465 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:35:21,465 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,465 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:21,466 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,467 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,467 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,469 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,469 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,470 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:35:21,471 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:21,474 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:35:21,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859339, jitterRate=0.09270656108856201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:35:21,476 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732296921453Initializing all the Stores at 1732296921454 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296921454Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296921454Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296921454Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296921454Cleaning up temporary data from old regions at 1732296921469 (+15 ms)Region opened successfully at 1732296921476 (+7 ms)
2024-11-22T17:35:21,477 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:35:21,481 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba49c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:35:21,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:35:21,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:35:21,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:35:21,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:35:21,483 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:35:21,483 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:35:21,483 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:35:21,486 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:35:21,487 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:35:21,534 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:35:21,534 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:35:21,535 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:35:21,548 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:35:21,548 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:35:21,550 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:35:21,558 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:35:21,560 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:35:21,569 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:35:21,574 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:35:21,586 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:35:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,599 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,32931,1732296920691, sessionid=0x101639d4a0b0000, setting cluster-up flag (Was=false)
2024-11-22T17:35:21,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,654 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:35:21,659 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,32931,1732296920691
2024-11-22T17:35:21,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:21,716 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:35:21,719 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,32931,1732296920691
2024-11-22T17:35:21,722 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:35:21,727 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:21,728 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:35:21,728 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:35:21,729 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,32931,1732296920691
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:35:21,732 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:35:21,733 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,736 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:21,736 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:35:21,736 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732296951736
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:35:21,737 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,738 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,738 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:35:21,738 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:35:21,738 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:35:21,738 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:35:21,738 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:35:21,738 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:35:21,738 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296921738,5,FailOnTimeoutGroup]
2024-11-22T17:35:21,740 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296921738,5,FailOnTimeoutGroup]
2024-11-22T17:35:21,740 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,740 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:35:21,740 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,740 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:35:21,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:35:21,748 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:35:21,748 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe
2024-11-22T17:35:21,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:35:21,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:35:21,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:21,759 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:35:21,761 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:35:21,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,761 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:21,761 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:35:21,763 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:35:21,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:21,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:35:21,765 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:35:21,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,765 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:21,765 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:35:21,767 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:35:21,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:21,768 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:21,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:35:21,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740
2024-11-22T17:35:21,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740
2024-11-22T17:35:21,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:35:21,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:35:21,771 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:35:21,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:35:21,774 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:35:21,775 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=883086, jitterRate=0.12290214002132416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:35:21,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732296921758Initializing all the Stores at 1732296921758Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296921759 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296921759Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296921759Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296921759Cleaning up temporary data from old regions at 1732296921771 (+12 ms)Region opened successfully at 1732296921776 (+5 ms)
2024-11-22T17:35:21,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:35:21,776 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:35:21,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:35:21,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:35:21,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:35:21,777 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:21,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296921776Disabling compacts and flushes for region at 1732296921776Disabling writes for close at 1732296921776Writing region close event to WAL at 1732296921777 (+1 ms)Closed at 1732296921777
2024-11-22T17:35:21,779 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:21,779 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:35:21,779 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:35:21,781 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:35:21,782 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:35:21,796 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(746): ClusterId : e60fcdae-4b2f-4758-b562-a011fbcfaa93
2024-11-22T17:35:21,796 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:35:21,809 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:35:21,809 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:35:21,823 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:35:21,823 DEBUG [RS:0;e61df7af61d2:46541 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388cfa43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:35:21,834 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:46541
2024-11-22T17:35:21,834 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:35:21,834 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:35:21,835 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:35:21,836 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,32931,1732296920691 with port=46541, startcode=1732296920861
2024-11-22T17:35:21,836 DEBUG [RS:0;e61df7af61d2:46541 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:35:21,838 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:35:21,839 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32931 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,839 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32931 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,842 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe
2024-11-22T17:35:21,842 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37617
2024-11-22T17:35:21,842 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:35:21,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:35:21,854 DEBUG [RS:0;e61df7af61d2:46541 {}] zookeeper.ZKUtil(111): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,854 WARN  [RS:0;e61df7af61d2:46541 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:35:21,854 INFO  [RS:0;e61df7af61d2:46541 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:21,854 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/WALs/e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,854 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,46541,1732296920861]
2024-11-22T17:35:21,859 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:35:21,861 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:35:21,861 INFO  [RS:0;e61df7af61d2:46541 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:35:21,861 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,861 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:35:21,862 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:35:21,863 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,863 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:21,864 DEBUG [RS:0;e61df7af61d2:46541 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:21,865 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,866 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,866 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,866 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,866 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,866 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296920861-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:35:21,881 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:35:21,882 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,46541,1732296920861-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,882 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,882 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.Replication(171): e61df7af61d2,46541,1732296920861 started
2024-11-22T17:35:21,895 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:21,895 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,46541,1732296920861, RpcServer on e61df7af61d2/172.17.0.2:46541, sessionid=0x101639d4a0b0001
2024-11-22T17:35:21,895 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:35:21,895 DEBUG [RS:0;e61df7af61d2:46541 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,895 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,46541,1732296920861'
2024-11-22T17:35:21,895 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:35:21,896 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:35:21,896 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:35:21,897 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:35:21,897 DEBUG [RS:0;e61df7af61d2:46541 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,46541,1732296920861
2024-11-22T17:35:21,897 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,46541,1732296920861'
2024-11-22T17:35:21,897 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:35:21,897 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:35:21,898 DEBUG [RS:0;e61df7af61d2:46541 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:35:21,898 INFO  [RS:0;e61df7af61d2:46541 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:35:21,898 INFO  [RS:0;e61df7af61d2:46541 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:35:21,932 WARN  [e61df7af61d2:32931 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:35:22,002 INFO  [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C46541%2C1732296920861, suffix=, logDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/WALs/e61df7af61d2,46541,1732296920861, archiveDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/oldWALs, maxLogs=32
2024-11-22T17:35:22,005 INFO  [RS:0;e61df7af61d2:46541 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C46541%2C1732296920861.1732296922005
2024-11-22T17:35:22,013 INFO  [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/WALs/e61df7af61d2,46541,1732296920861/e61df7af61d2%2C46541%2C1732296920861.1732296922005
2024-11-22T17:35:22,015 DEBUG [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35453:35453),(127.0.0.1/127.0.0.1:33311:33311)]
2024-11-22T17:35:22,183 DEBUG [e61df7af61d2:32931 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:35:22,183 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,46541,1732296920861
2024-11-22T17:35:22,185 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,46541,1732296920861, state=OPENING
2024-11-22T17:35:22,196 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:22,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:22,239 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:35:22,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:22,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:22,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:22,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:22,252 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:35:22,252 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,46541,1732296920861}]
2024-11-22T17:35:22,406 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:35:22,409 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56695, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:35:22,414 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:35:22,414 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:22,417 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C46541%2C1732296920861.meta, suffix=.meta, logDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/WALs/e61df7af61d2,46541,1732296920861, archiveDir=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/oldWALs, maxLogs=32
2024-11-22T17:35:22,420 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C46541%2C1732296920861.meta.1732296922420.meta
2024-11-22T17:35:22,431 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/WALs/e61df7af61d2,46541,1732296920861/e61df7af61d2%2C46541%2C1732296920861.meta.1732296922420.meta
2024-11-22T17:35:22,437 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35453:35453),(127.0.0.1/127.0.0.1:33311:33311)]
2024-11-22T17:35:22,441 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:35:22,441 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:35:22,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:35:22,442 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:35:22,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:35:22,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:22,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:35:22,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:35:22,445 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:35:22,447 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:35:22,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:22,448 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:22,448 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:35:22,450 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:35:22,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:22,454 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:22,455 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:35:22,457 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:35:22,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:22,461 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:22,461 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:35:22,463 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:35:22,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:22,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:22,464 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:35:22,466 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740
2024-11-22T17:35:22,468 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740
2024-11-22T17:35:22,471 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:35:22,471 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:35:22,475 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:35:22,478 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:35:22,480 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787463, jitterRate=0.0013112425804138184}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:35:22,481 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:35:22,483 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732296922443Writing region info on filesystem at 1732296922443Initializing all the Stores at 1732296922444 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296922444Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296922445 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296922445Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296922445Cleaning up temporary data from old regions at 1732296922471 (+26 ms)Running coprocessor post-open hooks at 1732296922481 (+10 ms)Region opened successfully at 1732296922483 (+2 ms)
2024-11-22T17:35:22,488 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732296922406
2024-11-22T17:35:22,493 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:35:22,493 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:35:22,494 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,46541,1732296920861
2024-11-22T17:35:22,496 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,46541,1732296920861, state=OPEN
2024-11-22T17:35:22,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:35:22,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:35:22,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:22,538 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,46541,1732296920861
2024-11-22T17:35:22,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:22,543 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:35:22,543 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,46541,1732296920861 in 286 msec
2024-11-22T17:35:22,549 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:35:22,549 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 764 msec
2024-11-22T17:35:22,551 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:22,551 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:35:22,554 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:35:22,554 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,46541,1732296920861, seqNum=-1]
2024-11-22T17:35:22,555 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:35:22,557 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39865, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:35:22,569 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 843 msec
2024-11-22T17:35:22,569 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732296922569, completionTime=-1
2024-11-22T17:35:22,570 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:35:22,570 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732296982573
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297042573
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,573 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,574 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:32931, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,574 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,574 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,577 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:35:22,587 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.638sec
2024-11-22T17:35:22,587 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:35:22,587 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:35:22,588 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:35:22,588 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:35:22,588 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:35:22,588 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:35:22,588 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:35:22,592 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:35:22,593 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:35:22,593 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,32931,1732296920691-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:22,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d21920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:22,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,32931,-1 for getting cluster id
2024-11-22T17:35:22,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:35:22,609 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e60fcdae-4b2f-4758-b562-a011fbcfaa93'
2024-11-22T17:35:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:35:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e60fcdae-4b2f-4758-b562-a011fbcfaa93"

2024-11-22T17:35:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6384b80b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,32931,-1]
2024-11-22T17:35:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:35:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,617 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:35:22,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b23b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:22,619 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:35:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,46541,1732296920861, seqNum=-1]
2024-11-22T17:35:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:35:22,626 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:35:22,630 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,32931,1732296920691
2024-11-22T17:35:22,631 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:22,637 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:35:22,638 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:35:22,638 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:35:22,638 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:22,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,639 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:35:22,639 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:35:22,639 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1972713769, stopped=false
2024-11-22T17:35:22,639 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,32931,1732296920691
2024-11-22T17:35:22,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:22,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:22,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:22,660 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:35:22,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:22,660 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:35:22,661 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:22,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:22,662 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,46541,1732296920861' *****
2024-11-22T17:35:22,662 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:35:22,662 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:35:22,662 INFO  [RS:0;e61df7af61d2:46541 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:35:22,662 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:35:22,662 INFO  [RS:0;e61df7af61d2:46541 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:35:22,662 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:22,662 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,46541,1732296920861
2024-11-22T17:35:22,662 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:35:22,663 INFO  [RS:0;e61df7af61d2:46541 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:46541.
2024-11-22T17:35:22,663 DEBUG [RS:0;e61df7af61d2:46541 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:35:22,663 DEBUG [RS:0;e61df7af61d2:46541 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,663 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:35:22,663 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:35:22,663 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:35:22,663 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:35:22,668 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-22T17:35:22,668 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:35:22,668 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:35:22,668 DEBUG [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T17:35:22,669 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:35:22,669 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:35:22,669 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:35:22,669 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:35:22,669 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-22T17:35:22,699 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/.tmp/ns/930cc994eb0d4c0d8007f95d216e5ba5 is 43, key is default/ns:d/1732296922559/Put/seqid=0
2024-11-22T17:35:22,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741835_1011 (size=5153)
2024-11-22T17:35:22,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741835_1011 (size=5153)
2024-11-22T17:35:22,712 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/.tmp/ns/930cc994eb0d4c0d8007f95d216e5ba5
2024-11-22T17:35:22,715 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:35:22,722 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:22,723 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/.tmp/ns/930cc994eb0d4c0d8007f95d216e5ba5 as hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/ns/930cc994eb0d4c0d8007f95d216e5ba5
2024-11-22T17:35:22,734 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/ns/930cc994eb0d4c0d8007f95d216e5ba5, entries=2, sequenceid=6, filesize=5.0 K
2024-11-22T17:35:22,736 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 67ms, sequenceid=6, compaction requested=false
2024-11-22T17:35:22,736 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T17:35:22,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:22,754 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-22T17:35:22,757 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:35:22,757 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:22,757 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296922668Running coprocessor pre-close hooks at 1732296922668Disabling compacts and flushes for region at 1732296922668Disabling writes for close at 1732296922669 (+1 ms)Obtaining lock to block concurrent updates at 1732296922669Preparing flush snapshotting stores in 1588230740 at 1732296922669Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732296922670 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732296922671 (+1 ms)Flushing 1588230740/ns: creating writer at 1732296922671Flushing 1588230740/ns: appending metadata at 1732296922699 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1732296922699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fa57c0a: reopening flushed file at 1732296922722 (+23 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 67ms, sequenceid=6, compaction requested=false at 1732296922736 (+14 ms)Writing region close event to WAL at 1732296922748 (+12 ms)Running coprocessor post-close hooks at 1732296922757 (+9 ms)Closed at 1732296922757
2024-11-22T17:35:22,758 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:22,869 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,46541,1732296920861; all regions closed.
2024-11-22T17:35:22,869 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,870 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,870 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,870 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,870 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741834_1010 (size=1152)
2024-11-22T17:35:22,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741834_1010 (size=1152)
2024-11-22T17:35:22,875 DEBUG [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/oldWALs
2024-11-22T17:35:22,875 INFO  [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C46541%2C1732296920861.meta:.meta(num 1732296922420)
2024-11-22T17:35:22,876 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,876 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,876 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,876 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,876 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:22,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741833_1009 (size=93)
2024-11-22T17:35:22,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741833_1009 (size=93)
2024-11-22T17:35:22,880 DEBUG [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/oldWALs
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C46541%2C1732296920861:(num 1732296922005)
2024-11-22T17:35:22,881 DEBUG [RS:0;e61df7af61d2:46541 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:35:22,881 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:35:22,881 INFO  [RS:0;e61df7af61d2:46541 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46541
2024-11-22T17:35:22,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:35:22,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,46541,1732296920861
2024-11-22T17:35:22,892 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:35:22,892 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,46541,1732296920861]
2024-11-22T17:35:22,912 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,46541,1732296920861 already deleted, retry=false
2024-11-22T17:35:22,913 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,46541,1732296920861 expired; onlineServers=0
2024-11-22T17:35:22,913 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,32931,1732296920691' *****
2024-11-22T17:35:22,913 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:35:22,913 INFO  [M:0;e61df7af61d2:32931 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:35:22,913 INFO  [M:0;e61df7af61d2:32931 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:35:22,913 DEBUG [M:0;e61df7af61d2:32931 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:35:22,913 DEBUG [M:0;e61df7af61d2:32931 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:35:22,913 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:35:22,913 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296921738 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296921738,5,FailOnTimeoutGroup]
2024-11-22T17:35:22,913 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296921738 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296921738,5,FailOnTimeoutGroup]
2024-11-22T17:35:22,913 INFO  [M:0;e61df7af61d2:32931 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:35:22,913 INFO  [M:0;e61df7af61d2:32931 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:35:22,914 DEBUG [M:0;e61df7af61d2:32931 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:35:22,914 INFO  [M:0;e61df7af61d2:32931 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:35:22,914 INFO  [M:0;e61df7af61d2:32931 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:35:22,914 INFO  [M:0;e61df7af61d2:32931 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:35:22,914 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:35:22,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:35:22,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:22,923 DEBUG [M:0;e61df7af61d2:32931 {}] zookeeper.ZKUtil(347): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:35:22,923 WARN  [M:0;e61df7af61d2:32931 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:35:22,924 INFO  [M:0;e61df7af61d2:32931 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/.lastflushedseqids
2024-11-22T17:35:22,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741836_1012 (size=99)
2024-11-22T17:35:22,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741836_1012 (size=99)
2024-11-22T17:35:22,933 INFO  [M:0;e61df7af61d2:32931 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:35:22,933 INFO  [M:0;e61df7af61d2:32931 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:35:22,933 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:35:22,933 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:22,933 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:22,934 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:35:22,934 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:22,934 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-22T17:35:22,954 DEBUG [M:0;e61df7af61d2:32931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d09e0b33b4e1451fb66e21d9ee547998 is 82, key is hbase:meta,,1/info:regioninfo/1732296922494/Put/seqid=0
2024-11-22T17:35:22,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741837_1013 (size=5672)
2024-11-22T17:35:22,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741837_1013 (size=5672)
2024-11-22T17:35:22,960 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d09e0b33b4e1451fb66e21d9ee547998
2024-11-22T17:35:22,988 DEBUG [M:0;e61df7af61d2:32931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0d9f3866d114680a14fe8610c4fdc2a is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732296922568/Put/seqid=0
2024-11-22T17:35:22,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741838_1014 (size=5275)
2024-11-22T17:35:22,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741838_1014 (size=5275)
2024-11-22T17:35:22,994 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0d9f3866d114680a14fe8610c4fdc2a
2024-11-22T17:35:23,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:23,002 INFO  [RS:0;e61df7af61d2:46541 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:35:23,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46541-0x101639d4a0b0001, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:23,002 INFO  [RS:0;e61df7af61d2:46541 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,46541,1732296920861; zookeeper connection closed.
2024-11-22T17:35:23,003 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@379ec9eb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@379ec9eb
2024-11-22T17:35:23,003 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:35:23,015 DEBUG [M:0;e61df7af61d2:32931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/719c667b7178439b8a4ea4121b2a8e79 is 69, key is e61df7af61d2,46541,1732296920861/rs:state/1732296921839/Put/seqid=0
2024-11-22T17:35:23,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741839_1015 (size=5156)
2024-11-22T17:35:23,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741839_1015 (size=5156)
2024-11-22T17:35:23,021 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/719c667b7178439b8a4ea4121b2a8e79
2024-11-22T17:35:23,044 DEBUG [M:0;e61df7af61d2:32931 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d896c8c81cf4b8ea9280ddc33ccd5fc is 52, key is load_balancer_on/state:d/1732296922635/Put/seqid=0
2024-11-22T17:35:23,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741840_1016 (size=5056)
2024-11-22T17:35:23,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741840_1016 (size=5056)
2024-11-22T17:35:23,049 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d896c8c81cf4b8ea9280ddc33ccd5fc
2024-11-22T17:35:23,057 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d09e0b33b4e1451fb66e21d9ee547998 as hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d09e0b33b4e1451fb66e21d9ee547998
2024-11-22T17:35:23,064 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d09e0b33b4e1451fb66e21d9ee547998, entries=8, sequenceid=29, filesize=5.5 K
2024-11-22T17:35:23,065 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d0d9f3866d114680a14fe8610c4fdc2a as hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d0d9f3866d114680a14fe8610c4fdc2a
2024-11-22T17:35:23,071 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d0d9f3866d114680a14fe8610c4fdc2a, entries=3, sequenceid=29, filesize=5.2 K
2024-11-22T17:35:23,073 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/719c667b7178439b8a4ea4121b2a8e79 as hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/719c667b7178439b8a4ea4121b2a8e79
2024-11-22T17:35:23,079 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/719c667b7178439b8a4ea4121b2a8e79, entries=1, sequenceid=29, filesize=5.0 K
2024-11-22T17:35:23,080 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9d896c8c81cf4b8ea9280ddc33ccd5fc as hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d896c8c81cf4b8ea9280ddc33ccd5fc
2024-11-22T17:35:23,086 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37617/user/jenkins/test-data/2cefdc32-0c5e-55a4-bf2b-004352c1dcbe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9d896c8c81cf4b8ea9280ddc33ccd5fc, entries=1, sequenceid=29, filesize=4.9 K
2024-11-22T17:35:23,088 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=29, compaction requested=false
2024-11-22T17:35:23,089 INFO  [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:23,089 DEBUG [M:0;e61df7af61d2:32931 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296922933Disabling compacts and flushes for region at 1732296922933Disabling writes for close at 1732296922934 (+1 ms)Obtaining lock to block concurrent updates at 1732296922934Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732296922934Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732296922934Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732296922935 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732296922935Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732296922953 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732296922953Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732296922967 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732296922987 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732296922987Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732296923000 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732296923015 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732296923015Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732296923026 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732296923043 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732296923043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@718ff707: reopening flushed file at 1732296923056 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@728bd81c: reopening flushed file at 1732296923064 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a193964: reopening flushed file at 1732296923072 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6604fcdb: reopening flushed file at 1732296923079 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=29, compaction requested=false at 1732296923088 (+9 ms)Writing region close event to WAL at 1732296923089 (+1 ms)Closed at 1732296923089
2024-11-22T17:35:23,090 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:23,090 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:23,090 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:23,090 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:23,090 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:23,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44933 is added to blk_1073741830_1006 (size=10311)
2024-11-22T17:35:23,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40487 is added to blk_1073741830_1006 (size=10311)
2024-11-22T17:35:23,093 INFO  [M:0;e61df7af61d2:32931 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:35:23,093 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:35:23,093 INFO  [M:0;e61df7af61d2:32931 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32931
2024-11-22T17:35:23,093 INFO  [M:0;e61df7af61d2:32931 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:35:23,206 INFO  [M:0;e61df7af61d2:32931 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:35:23,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:23,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32931-0x101639d4a0b0000, quorum=127.0.0.1:50743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:35:23,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@513d1177{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:23,211 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c0a2992{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:23,211 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:23,211 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1db3e806{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:23,212 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c5ed954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:23,214 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:23,214 WARN  [BP-846298224-172.17.0.2-1732296918235 heartbeating to localhost/127.0.0.1:37617 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:23,214 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:23,214 WARN  [BP-846298224-172.17.0.2-1732296918235 heartbeating to localhost/127.0.0.1:37617 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-846298224-172.17.0.2-1732296918235 (Datanode Uuid ca20f399-658e-42bc-8894-5e2cc30fc9b1) service to localhost/127.0.0.1:37617
2024-11-22T17:35:23,216 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data3/current/BP-846298224-172.17.0.2-1732296918235 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:23,216 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data4/current/BP-846298224-172.17.0.2-1732296918235 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:23,216 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:23,220 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a843c6e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:23,221 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17783943{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:23,221 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:23,221 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4bedb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:23,221 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a3a779{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:23,224 WARN  [BP-846298224-172.17.0.2-1732296918235 heartbeating to localhost/127.0.0.1:37617 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:23,224 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:23,224 WARN  [BP-846298224-172.17.0.2-1732296918235 heartbeating to localhost/127.0.0.1:37617 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-846298224-172.17.0.2-1732296918235 (Datanode Uuid 4f40fd81-c871-43ed-9746-b6f53c0601ff) service to localhost/127.0.0.1:37617
2024-11-22T17:35:23,224 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:23,225 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data1/current/BP-846298224-172.17.0.2-1732296918235 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:23,226 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/cluster_51f6c9d9-5bee-9238-b773-f3248f09d7ac/data/data2/current/BP-846298224-172.17.0.2-1732296918235 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:23,226 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:23,231 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2787d4b6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:35:23,232 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f450072{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:23,232 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:23,232 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@495543c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:23,232 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c8f0dfe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:23,238 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:35:23,253 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.log.dir so I do NOT create it in target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0df040fc-bdd5-7224-3358-1175fa48ed05/hadoop.tmp.dir so I do NOT create it in target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5, deleteOnExit=true
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/test.cache.data in system properties and HBase conf
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:35:23,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:35:23,255 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:35:23,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:35:23,256 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:35:23,268 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:35:23,579 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:23,584 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:23,585 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:23,585 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:23,585 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:35:23,586 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:23,586 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37564f36{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:23,587 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ff0d030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:23,683 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d90daca{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-38069-hadoop-hdfs-3_4_1-tests_jar-_-any-8212224935246637182/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:35:23,684 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16ad4404{HTTP/1.1, (http/1.1)}{localhost:38069}
2024-11-22T17:35:23,684 INFO  [Time-limited test {}] server.Server(415): Started @109721ms
2024-11-22T17:35:23,696 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:35:23,868 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:35:24,132 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:24,135 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:24,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:24,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:24,136 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:35:24,136 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@172c9107{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:24,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6425d7f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:24,229 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dfaace0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-37905-hadoop-hdfs-3_4_1-tests_jar-_-any-4466835059617767963/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:24,230 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@596eb72c{HTTP/1.1, (http/1.1)}{localhost:37905}
2024-11-22T17:35:24,230 INFO  [Time-limited test {}] server.Server(415): Started @110268ms
2024-11-22T17:35:24,231 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:24,261 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:24,265 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:24,266 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:24,266 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:24,266 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:35:24,266 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3372b5da{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:24,267 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56914f2c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:24,359 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3d88de8c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-43471-hadoop-hdfs-3_4_1-tests_jar-_-any-11667568990912500951/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:24,359 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@36318c9a{HTTP/1.1, (http/1.1)}{localhost:43471}
2024-11-22T17:35:24,359 INFO  [Time-limited test {}] server.Server(415): Started @110397ms
2024-11-22T17:35:24,360 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:25,387 WARN  [Thread-671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data1/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:25,387 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data2/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:25,405 WARN  [Thread-635 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:25,407 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x777f112f2efb36e6 with lease ID 0x86a82708ce34249f: Processing first storage report for DS-92bbc77f-f18e-49e2-8039-5ee474d78577 from datanode DatanodeRegistration(127.0.0.1:46859, datanodeUuid=cdd6016f-30e9-4010-a8ca-29dba561881b, infoPort=33423, infoSecurePort=0, ipcPort=32977, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:25,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x777f112f2efb36e6 with lease ID 0x86a82708ce34249f: from storage DS-92bbc77f-f18e-49e2-8039-5ee474d78577 node DatanodeRegistration(127.0.0.1:46859, datanodeUuid=cdd6016f-30e9-4010-a8ca-29dba561881b, infoPort=33423, infoSecurePort=0, ipcPort=32977, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:25,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x777f112f2efb36e6 with lease ID 0x86a82708ce34249f: Processing first storage report for DS-6c308c37-b046-4d27-91f2-fca39f030589 from datanode DatanodeRegistration(127.0.0.1:46859, datanodeUuid=cdd6016f-30e9-4010-a8ca-29dba561881b, infoPort=33423, infoSecurePort=0, ipcPort=32977, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:25,408 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x777f112f2efb36e6 with lease ID 0x86a82708ce34249f: from storage DS-6c308c37-b046-4d27-91f2-fca39f030589 node DatanodeRegistration(127.0.0.1:46859, datanodeUuid=cdd6016f-30e9-4010-a8ca-29dba561881b, infoPort=33423, infoSecurePort=0, ipcPort=32977, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:25,510 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:25,510 WARN  [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:25,533 WARN  [Thread-658 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:25,536 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd76ff419b4320a4e with lease ID 0x86a82708ce3424a0: Processing first storage report for DS-61e332e5-ddff-4342-ab2c-97e873a586bf from datanode DatanodeRegistration(127.0.0.1:44863, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=33097, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:25,536 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd76ff419b4320a4e with lease ID 0x86a82708ce3424a0: from storage DS-61e332e5-ddff-4342-ab2c-97e873a586bf node DatanodeRegistration(127.0.0.1:44863, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=33097, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:35:25,536 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd76ff419b4320a4e with lease ID 0x86a82708ce3424a0: Processing first storage report for DS-074d536e-7101-4627-b352-f41f96358366 from datanode DatanodeRegistration(127.0.0.1:44863, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=33097, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:25,536 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd76ff419b4320a4e with lease ID 0x86a82708ce3424a0: from storage DS-074d536e-7101-4627-b352-f41f96358366 node DatanodeRegistration(127.0.0.1:44863, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=33097, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:25,606 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee
2024-11-22T17:35:25,610 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/zookeeper_0, clientPort=63933, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:35:25,611 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63933
2024-11-22T17:35:25,612 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,614 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:35:25,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:35:25,628 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24 with version=8
2024-11-22T17:35:25,628 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:35:25,630 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:35:25,630 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:35:25,631 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:35:25,632 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42763
2024-11-22T17:35:25,633 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42763 connecting to ZooKeeper ensemble=127.0.0.1:63933
2024-11-22T17:35:25,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:427630x0, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:35:25,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42763-0x101639d5d540000 connected
2024-11-22T17:35:25,790 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,792 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:25,795 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24, hbase.cluster.distributed=false
2024-11-22T17:35:25,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:35:25,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42763
2024-11-22T17:35:25,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42763
2024-11-22T17:35:25,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42763
2024-11-22T17:35:25,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42763
2024-11-22T17:35:25,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42763
2024-11-22T17:35:25,813 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:35:25,814 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:35:25,815 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34069
2024-11-22T17:35:25,816 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34069 connecting to ZooKeeper ensemble=127.0.0.1:63933
2024-11-22T17:35:25,817 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,819 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340690x0, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:35:25,832 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:340690x0, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:35:25,832 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34069-0x101639d5d540001 connected
2024-11-22T17:35:25,833 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:35:25,833 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:35:25,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:35:25,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:35:25,835 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34069
2024-11-22T17:35:25,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34069
2024-11-22T17:35:25,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34069
2024-11-22T17:35:25,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34069
2024-11-22T17:35:25,837 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34069
2024-11-22T17:35:25,850 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:42763
2024-11-22T17:35:25,851 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,42763,1732296925630
2024-11-22T17:35:25,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:25,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:25,864 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,42763,1732296925630
2024-11-22T17:35:25,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:35:25,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:25,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:25,875 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:35:25,876 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,42763,1732296925630 from backup master directory
2024-11-22T17:35:25,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,42763,1732296925630
2024-11-22T17:35:25,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:25,884 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:35:25,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:35:25,885 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,42763,1732296925630
2024-11-22T17:35:25,891 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/hbase.id] with ID: 1c529e35-43cd-4e30-9dbd-721e11b31cc3
2024-11-22T17:35:25,891 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/.tmp/hbase.id
2024-11-22T17:35:25,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:35:25,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:35:25,901 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/.tmp/hbase.id]:[hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/hbase.id]
2024-11-22T17:35:25,915 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:25,915 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:35:25,916 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T17:35:25,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:25,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:25,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:35:25,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:35:25,939 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:35:25,940 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:35:25,941 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:25,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:35:25,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:35:25,950 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store
2024-11-22T17:35:25,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:35:25,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:35:25,958 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:25,958 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:35:25,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296925958Disabling compacts and flushes for region at 1732296925958Disabling writes for close at 1732296925958Writing region close event to WAL at 1732296925958Closed at 1732296925958
2024-11-22T17:35:25,959 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/.initializing
2024-11-22T17:35:25,959 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630
2024-11-22T17:35:25,963 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C42763%2C1732296925630, suffix=, logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630, archiveDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/oldWALs, maxLogs=10
2024-11-22T17:35:25,964 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C42763%2C1732296925630.1732296925963
2024-11-22T17:35:25,971 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963
2024-11-22T17:35:25,973 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33423:33423),(127.0.0.1/127.0.0.1:33097:33097)]
2024-11-22T17:35:25,974 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:35:25,974 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:25,974 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,974 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,976 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:25,977 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:35:25,978 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:25,978 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:25,978 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:25,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:35:25,980 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:25,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:25,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:25,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:35:25,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:25,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:25,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:35:25,984 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:35:25,984 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:25,984 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:25,984 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,985 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,986 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,988 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,988 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,989 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:35:25,990 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:35:25,994 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:35:25,994 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=843674, jitterRate=0.0727875679731369}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:35:25,996 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732296925974Initializing all the Stores at 1732296925975 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296925975Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296925976 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296925976Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296925976Cleaning up temporary data from old regions at 1732296925988 (+12 ms)Region opened successfully at 1732296925996 (+8 ms)
2024-11-22T17:35:25,996 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:35:26,001 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a6815a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:35:26,002 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:35:26,002 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:35:26,002 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:35:26,002 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:35:26,003 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:35:26,003 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:35:26,003 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:35:26,005 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:35:26,006 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:35:26,017 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:35:26,018 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:35:26,018 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:35:26,028 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:35:26,029 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:35:26,030 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:35:26,042 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:35:26,044 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:35:26,053 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:35:26,055 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:35:26,063 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:35:26,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:26,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:35:26,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,074 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,42763,1732296925630, sessionid=0x101639d5d540000, setting cluster-up flag (Was=false)
2024-11-22T17:35:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,127 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:35:26,128 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,42763,1732296925630
2024-11-22T17:35:26,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,179 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:35:26,181 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,42763,1732296925630
2024-11-22T17:35:26,182 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:35:26,185 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:26,185 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:35:26,185 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:35:26,185 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,42763,1732296925630
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:35:26,187 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:26,187 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:26,187 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:26,187 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:35:26,188 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:35:26,188 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,188 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:35:26,188 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732296956189
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:35:26,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:35:26,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:35:26,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:35:26,190 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:26,190 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:35:26,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:35:26,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:35:26,191 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296926191,5,FailOnTimeoutGroup]
2024-11-22T17:35:26,191 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296926191,5,FailOnTimeoutGroup]
2024-11-22T17:35:26,191 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,191 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:35:26,191 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,191 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,192 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,192 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:35:26,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:35:26,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:35:26,205 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:35:26,205 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24
2024-11-22T17:35:26,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:35:26,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:35:26,217 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:26,219 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:35:26,221 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:35:26,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,221 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,222 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:35:26,223 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:35:26,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,224 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,224 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:35:26,226 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:35:26,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,227 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:35:26,229 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:35:26,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,229 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:35:26,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740
2024-11-22T17:35:26,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740
2024-11-22T17:35:26,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:35:26,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:35:26,233 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:35:26,234 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:35:26,236 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:35:26,237 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715341, jitterRate=-0.09039784967899323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:35:26,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732296926217Initializing all the Stores at 1732296926218 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926218Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926218Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296926219 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926219Cleaning up temporary data from old regions at 1732296926232 (+13 ms)Region opened successfully at 1732296926237 (+5 ms)
2024-11-22T17:35:26,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:35:26,237 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:35:26,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:35:26,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:35:26,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:35:26,238 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:35:26,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296926237Disabling compacts and flushes for region at 1732296926237Disabling writes for close at 1732296926238 (+1 ms)Writing region close event to WAL at 1732296926238Closed at 1732296926238
2024-11-22T17:35:26,239 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(746): ClusterId : 1c529e35-43cd-4e30-9dbd-721e11b31cc3
2024-11-22T17:35:26,239 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:35:26,240 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:26,240 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:35:26,240 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:35:26,241 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:35:26,243 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:35:26,250 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:35:26,250 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:35:26,261 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:35:26,261 DEBUG [RS:0;e61df7af61d2:34069 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f27856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:35:26,303 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:34069
2024-11-22T17:35:26,303 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:35:26,303 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:35:26,303 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:35:26,304 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,42763,1732296925630 with port=34069, startcode=1732296925813
2024-11-22T17:35:26,304 DEBUG [RS:0;e61df7af61d2:34069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:35:26,307 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46027, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:35:26,307 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42763 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,307 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42763 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,309 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24
2024-11-22T17:35:26,309 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33455
2024-11-22T17:35:26,309 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:35:26,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:35:26,323 DEBUG [RS:0;e61df7af61d2:34069 {}] zookeeper.ZKUtil(111): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,323 WARN  [RS:0;e61df7af61d2:34069 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:35:26,324 INFO  [RS:0;e61df7af61d2:34069 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:26,324 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,324 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,34069,1732296925813]
2024-11-22T17:35:26,336 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:35:26,338 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:35:26,340 INFO  [RS:0;e61df7af61d2:34069 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:35:26,340 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,341 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:35:26,342 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:35:26,342 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,342 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,342 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,342 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,342 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,342 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:26,343 DEBUG [RS:0;e61df7af61d2:34069 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,344 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34069,1732296925813-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:35:26,358 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:35:26,358 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34069,1732296925813-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,358 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,358 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.Replication(171): e61df7af61d2,34069,1732296925813 started
2024-11-22T17:35:26,370 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,370 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,34069,1732296925813, RpcServer on e61df7af61d2/172.17.0.2:34069, sessionid=0x101639d5d540001
2024-11-22T17:35:26,370 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:35:26,370 DEBUG [RS:0;e61df7af61d2:34069 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,370 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,34069,1732296925813'
2024-11-22T17:35:26,370 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,34069,1732296925813'
2024-11-22T17:35:26,371 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:35:26,372 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:35:26,373 DEBUG [RS:0;e61df7af61d2:34069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:35:26,373 INFO  [RS:0;e61df7af61d2:34069 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:35:26,373 INFO  [RS:0;e61df7af61d2:34069 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:35:26,393 WARN  [e61df7af61d2:42763 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:35:26,478 INFO  [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C34069%2C1732296925813, suffix=, logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813, archiveDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs, maxLogs=32
2024-11-22T17:35:26,480 INFO  [RS:0;e61df7af61d2:34069 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296926480
2024-11-22T17:35:26,489 INFO  [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480
2024-11-22T17:35:26,491 DEBUG [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33423:33423),(127.0.0.1/127.0.0.1:33097:33097)]
2024-11-22T17:35:26,643 DEBUG [e61df7af61d2:42763 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:35:26,645 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,649 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,34069,1732296925813, state=OPENING
2024-11-22T17:35:26,702 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:35:26,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:35:26,714 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:35:26,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:26,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:26,715 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,34069,1732296925813}]
2024-11-22T17:35:26,870 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:35:26,871 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42877, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:35:26,876 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:35:26,876 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:26,878 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C34069%2C1732296925813.meta, suffix=.meta, logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813, archiveDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs, maxLogs=32
2024-11-22T17:35:26,879 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
2024-11-22T17:35:26,884 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
2024-11-22T17:35:26,888 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33097:33097),(127.0.0.1/127.0.0.1:33423:33423)]
2024-11-22T17:35:26,889 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:35:26,890 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:35:26,890 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:35:26,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:35:26,892 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:35:26,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,893 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,893 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:35:26,894 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:35:26,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:35:26,895 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:35:26,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,895 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,896 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:35:26,896 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:35:26,896 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:26,897 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:35:26,897 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:35:26,897 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740
2024-11-22T17:35:26,899 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740
2024-11-22T17:35:26,900 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:35:26,900 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:35:26,900 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:35:26,902 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:35:26,902 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=819397, jitterRate=0.04191772639751434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:35:26,903 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:35:26,903 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732296926890Writing region info on filesystem at 1732296926890Initializing all the Stores at 1732296926891 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926891Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926891Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296926891Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296926891Cleaning up temporary data from old regions at 1732296926900 (+9 ms)Running coprocessor post-open hooks at 1732296926903 (+3 ms)Region opened successfully at 1732296926903
2024-11-22T17:35:26,904 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732296926869
2024-11-22T17:35:26,907 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:35:26,907 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:35:26,908 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,909 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,34069,1732296925813, state=OPEN
2024-11-22T17:35:26,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:35:26,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:35:26,946 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:26,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:26,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:35:26,950 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:35:26,950 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,34069,1732296925813 in 231 msec
2024-11-22T17:35:26,953 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:35:26,953 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 710 msec
2024-11-22T17:35:26,954 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:35:26,954 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:35:26,955 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:35:26,955 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,34069,1732296925813, seqNum=-1]
2024-11-22T17:35:26,955 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:35:26,957 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52405, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:35:26,963 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 778 msec
2024-11-22T17:35:26,963 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732296926963, completionTime=-1
2024-11-22T17:35:26,963 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:35:26,963 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732296986965
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297046965
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:42763, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,965 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:26,967 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.084sec
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:35:26,969 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:35:26,972 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:35:26,972 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:35:26,972 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,42763,1732296925630-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7663cfa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:27,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,42763,-1 for getting cluster id
2024-11-22T17:35:27,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:35:27,041 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1c529e35-43cd-4e30-9dbd-721e11b31cc3'
2024-11-22T17:35:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:35:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1c529e35-43cd-4e30-9dbd-721e11b31cc3"

2024-11-22T17:35:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@257195fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,42763,-1]
2024-11-22T17:35:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:35:27,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:35:27,045 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:35:27,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f58e21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:35:27,046 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:35:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,34069,1732296925813, seqNum=-1]
2024-11-22T17:35:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:35:27,050 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:35:27,052 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,42763,1732296925630
2024-11-22T17:35:27,052 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:27,055 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:35:27,068 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:35:27,069 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41147
2024-11-22T17:35:27,070 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41147 connecting to ZooKeeper ensemble=127.0.0.1:63933
2024-11-22T17:35:27,071 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:27,072 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:35:27,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411470x0, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:35:27,096 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:411470x0, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-22T17:35:27,096 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-22T17:35:27,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41147-0x101639d5d540002 connected
2024-11-22T17:35:27,096 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:35:27,097 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:35:27,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:35:27,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:35:27,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41147
2024-11-22T17:35:27,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41147
2024-11-22T17:35:27,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41147
2024-11-22T17:35:27,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41147
2024-11-22T17:35:27,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41147
2024-11-22T17:35:27,102 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(746): ClusterId : 1c529e35-43cd-4e30-9dbd-721e11b31cc3
2024-11-22T17:35:27,103 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:35:27,113 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:35:27,113 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:35:27,124 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:35:27,124 DEBUG [RS:1;e61df7af61d2:41147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3209b3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:35:27,134 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e61df7af61d2:41147
2024-11-22T17:35:27,134 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:35:27,134 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:35:27,134 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:35:27,135 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,42763,1732296925630 with port=41147, startcode=1732296927067
2024-11-22T17:35:27,135 DEBUG [RS:1;e61df7af61d2:41147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:35:27,137 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44291, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:35:27,137 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42763 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,137 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42763 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,139 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24
2024-11-22T17:35:27,139 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33455
2024-11-22T17:35:27,139 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:35:27,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:35:27,148 DEBUG [RS:1;e61df7af61d2:41147 {}] zookeeper.ZKUtil(111): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,148 WARN  [RS:1;e61df7af61d2:41147 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:35:27,148 INFO  [RS:1;e61df7af61d2:41147 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:35:27,148 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,41147,1732296927067]
2024-11-22T17:35:27,148 DEBUG [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,151 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:35:27,154 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:35:27,154 INFO  [RS:1;e61df7af61d2:41147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:35:27,154 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,154 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:35:27,156 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:35:27,156 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,156 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:27,157 DEBUG [RS:1;e61df7af61d2:41147 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,158 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,41147,1732296927067-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:35:27,173 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:35:27,174 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,41147,1732296927067-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,174 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,174 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.Replication(171): e61df7af61d2,41147,1732296927067 started
2024-11-22T17:35:27,187 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:35:27,187 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,41147,1732296927067, RpcServer on e61df7af61d2/172.17.0.2:41147, sessionid=0x101639d5d540002
2024-11-22T17:35:27,188 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:35:27,188 DEBUG [RS:1;e61df7af61d2:41147 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,188 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;e61df7af61d2:41147,5,FailOnTimeoutGroup]
2024-11-22T17:35:27,188 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,41147,1732296927067'
2024-11-22T17:35:27,188 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:35:27,188 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-22T17:35:27,188 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:35:27,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,41147,1732296927067
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,41147,1732296927067'
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:35:27,189 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:35:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,42763,1732296925630
2024-11-22T17:35:27,190 DEBUG [RS:1;e61df7af61d2:41147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:35:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d5e75bb
2024-11-22T17:35:27,190 INFO  [RS:1;e61df7af61d2:41147 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:35:27,190 INFO  [RS:1;e61df7af61d2:41147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:35:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T17:35:27,193 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T17:35:27,193 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T17:35:27,193 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T17:35:27,194 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:35:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T17:35:27,197 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T17:35:27,198 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:27,198 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-22T17:35:27,200 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T17:35:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:35:27,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741835_1011 (size=393)
2024-11-22T17:35:27,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741835_1011 (size=393)
2024-11-22T17:35:27,212 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b8a95d7bc5ab19d1e8c0d41584d89ec, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24
2024-11-22T17:35:27,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44863 is added to blk_1073741836_1012 (size=76)
2024-11-22T17:35:27,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46859 is added to blk_1073741836_1012 (size=76)
2024-11-22T17:35:27,293 INFO  [RS:1;e61df7af61d2:41147 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C41147%2C1732296927067, suffix=, logDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067, archiveDir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs, maxLogs=32
2024-11-22T17:35:27,294 INFO  [RS:1;e61df7af61d2:41147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C41147%2C1732296927067.1732296927294
2024-11-22T17:35:27,302 INFO  [RS:1;e61df7af61d2:41147 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
2024-11-22T17:35:27,305 DEBUG [RS:1;e61df7af61d2:41147 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33097:33097),(127.0.0.1/127.0.0.1:33423:33423)]
2024-11-22T17:35:27,623 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:27,624 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 1b8a95d7bc5ab19d1e8c0d41584d89ec, disabling compactions & flushes
2024-11-22T17:35:27,624 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,624 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,624 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec. after waiting 0 ms
2024-11-22T17:35:27,624 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,624 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,624 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:
Waiting for close lock at 1732296927624Disabling compacts and flushes for region at 1732296927624Disabling writes for close at 1732296927624Writing region close event to WAL at 1732296927624Closed at 1732296927624
2024-11-22T17:35:27,627 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T17:35:27,628 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732296927627"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732296927627"}]},"ts":"1732296927627"}
2024-11-22T17:35:27,632 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T17:35:27,634 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T17:35:27,634 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296927634"}]},"ts":"1732296927634"}
2024-11-22T17:35:27,637 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-22T17:35:27,641 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {e61df7af61d2=0} racks are {/default-rack=0}
2024-11-22T17:35:27,646 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-22T17:35:27,646 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-22T17:35:27,646 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-22T17:35:27,646 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-22T17:35:27,646 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-22T17:35:27,646 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-22T17:35:27,646 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-22T17:35:27,648 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1b8a95d7bc5ab19d1e8c0d41584d89ec, ASSIGN}]
2024-11-22T17:35:27,650 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1b8a95d7bc5ab19d1e8c0d41584d89ec, ASSIGN
2024-11-22T17:35:27,651 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1b8a95d7bc5ab19d1e8c0d41584d89ec, ASSIGN; state=OFFLINE, location=e61df7af61d2,34069,1732296925813; forceNewPlan=false, retain=false
2024-11-22T17:35:27,802 INFO  [e61df7af61d2:42763 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-22T17:35:27,803 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b8a95d7bc5ab19d1e8c0d41584d89ec, regionState=OPENING, regionLocation=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:27,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1b8a95d7bc5ab19d1e8c0d41584d89ec, ASSIGN because future has completed
2024-11-22T17:35:27,806 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8a95d7bc5ab19d1e8c0d41584d89ec, server=e61df7af61d2,34069,1732296925813}]
2024-11-22T17:35:27,944 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:35:27,947 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:27,965 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,966 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b8a95d7bc5ab19d1e8c0d41584d89ec, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:35:27,966 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,967 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:35:27,967 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,967 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,967 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:27,970 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:27,971 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:27,972 INFO  [StoreOpener-1b8a95d7bc5ab19d1e8c0d41584d89ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1b8a95d7bc5ab19d1e8c0d41584d89ec 
2024-11-22T17:35:27,975 INFO  [StoreOpener-1b8a95d7bc5ab19d1e8c0d41584d89ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b8a95d7bc5ab19d1e8c0d41584d89ec columnFamilyName info
2024-11-22T17:35:27,975 DEBUG [StoreOpener-1b8a95d7bc5ab19d1e8c0d41584d89ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:35:27,976 INFO  [StoreOpener-1b8a95d7bc5ab19d1e8c0d41584d89ec-1 {}] regionserver.HStore(327): Store=1b8a95d7bc5ab19d1e8c0d41584d89ec/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:35:27,976 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,978 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,978 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,979 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,979 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,983 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,986 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:35:27,987 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1b8a95d7bc5ab19d1e8c0d41584d89ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868934, jitterRate=0.10490685701370239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:35:27,988 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:27,988 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:
Running coprocessor pre-open hook at 1732296927967Writing region info on filesystem at 1732296927967Initializing all the Stores at 1732296927968 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296927968Cleaning up temporary data from old regions at 1732296927979 (+11 ms)Running coprocessor post-open hooks at 1732296927988 (+9 ms)Region opened successfully at 1732296927988
2024-11-22T17:35:27,990 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., pid=6, masterSystemTime=1732296927960
2024-11-22T17:35:27,994 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,994 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:27,994 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b8a95d7bc5ab19d1e8c0d41584d89ec, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,34069,1732296925813
2024-11-22T17:35:27,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b8a95d7bc5ab19d1e8c0d41584d89ec, server=e61df7af61d2,34069,1732296925813 because future has completed
2024-11-22T17:35:28,004 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T17:35:28,004 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b8a95d7bc5ab19d1e8c0d41584d89ec, server=e61df7af61d2,34069,1732296925813 in 194 msec
2024-11-22T17:35:28,008 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T17:35:28,008 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=1b8a95d7bc5ab19d1e8c0d41584d89ec, ASSIGN in 356 msec
2024-11-22T17:35:28,010 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T17:35:28,010 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296928010"}]},"ts":"1732296928010"}
2024-11-22T17:35:28,013 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-22T17:35:28,015 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T17:35:28,018 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 821 msec
2024-11-22T17:35:31,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:35:31,205 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T17:35:31,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T17:35:31,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-22T17:35:31,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:35:31,207 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T17:35:31,208 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:35:31,208 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T17:35:32,336 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-22T17:35:33,470 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:35:33,472 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:33,506 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:33,512 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:33,513 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:35:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:35:37,254 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-22T17:35:37,254 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-22T17:35:37,258 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T17:35:37,258 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:37,276 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:37,281 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:37,281 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:37,282 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:37,282 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:35:37,282 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41b3c9b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:37,283 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7857b965{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:37,391 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@57d162c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-36685-hadoop-hdfs-3_4_1-tests_jar-_-any-5787896530993553083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:37,392 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d72813f{HTTP/1.1, (http/1.1)}{localhost:36685}
2024-11-22T17:35:37,392 INFO  [Time-limited test {}] server.Server(415): Started @123430ms
2024-11-22T17:35:37,393 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:37,431 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:37,434 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:37,435 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:37,435 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:37,435 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:35:37,436 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d7d08f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:37,436 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24a7a164{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:37,546 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@491335d6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-45649-hadoop-hdfs-3_4_1-tests_jar-_-any-5701210717541173517/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:37,547 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74b1788d{HTTP/1.1, (http/1.1)}{localhost:45649}
2024-11-22T17:35:37,547 INFO  [Time-limited test {}] server.Server(415): Started @123585ms
2024-11-22T17:35:37,548 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:37,592 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:37,595 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:37,596 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:37,596 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:37,596 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:35:37,597 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75748ead{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:37,597 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bbb3640{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:37,736 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@312f47e7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-33155-hadoop-hdfs-3_4_1-tests_jar-_-any-3404727164955565515/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:37,737 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20f3e03{HTTP/1.1, (http/1.1)}{localhost:33155}
2024-11-22T17:35:37,737 INFO  [Time-limited test {}] server.Server(415): Started @123775ms
2024-11-22T17:35:37,739 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:38,784 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:38,784 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:38,808 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:38,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5a2b8415853ce19 with lease ID 0x86a82708ce3424a1: Processing first storage report for DS-7c760086-3f6c-492f-b610-ce53af0d973e from datanode DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:38,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5a2b8415853ce19 with lease ID 0x86a82708ce3424a1: from storage DS-7c760086-3f6c-492f-b610-ce53af0d973e node DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:38,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5a2b8415853ce19 with lease ID 0x86a82708ce3424a1: Processing first storage report for DS-00b301a2-816f-46cf-8f5c-a59331ed6a73 from datanode DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:38,811 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5a2b8415853ce19 with lease ID 0x86a82708ce3424a1: from storage DS-00b301a2-816f-46cf-8f5c-a59331ed6a73 node DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:35:38,905 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data7/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:38,905 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data8/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:38,925 WARN  [Thread-830 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:38,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2119163f0834fa82 with lease ID 0x86a82708ce3424a2: Processing first storage report for DS-09a5c403-50f4-471b-aea9-64034f21cb30 from datanode DatanodeRegistration(127.0.0.1:38665, datanodeUuid=462da758-2675-48dc-aa1d-756e2ebc551a, infoPort=43873, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:38,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2119163f0834fa82 with lease ID 0x86a82708ce3424a2: from storage DS-09a5c403-50f4-471b-aea9-64034f21cb30 node DatanodeRegistration(127.0.0.1:38665, datanodeUuid=462da758-2675-48dc-aa1d-756e2ebc551a, infoPort=43873, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:35:38,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2119163f0834fa82 with lease ID 0x86a82708ce3424a2: Processing first storage report for DS-c070c7f5-4241-4344-9e04-a26c4f35dab9 from datanode DatanodeRegistration(127.0.0.1:38665, datanodeUuid=462da758-2675-48dc-aa1d-756e2ebc551a, infoPort=43873, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:38,927 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2119163f0834fa82 with lease ID 0x86a82708ce3424a2: from storage DS-c070c7f5-4241-4344-9e04-a26c4f35dab9 node DatanodeRegistration(127.0.0.1:38665, datanodeUuid=462da758-2675-48dc-aa1d-756e2ebc551a, infoPort=43873, infoSecurePort=0, ipcPort=39345, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:39,077 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data9/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:39,077 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data10/current/BP-1602666847-172.17.0.2-1732296923279/current, will proceed with Du for space computation calculation, 
2024-11-22T17:35:39,096 WARN  [Thread-852 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:39,098 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe55641d9713983ab with lease ID 0x86a82708ce3424a3: Processing first storage report for DS-8397671a-79e3-4afd-9529-1f1a5de0b16d from datanode DatanodeRegistration(127.0.0.1:40373, datanodeUuid=5966ade0-7b9a-4ced-87d7-5772caa42bfa, infoPort=46493, infoSecurePort=0, ipcPort=38205, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:39,099 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe55641d9713983ab with lease ID 0x86a82708ce3424a3: from storage DS-8397671a-79e3-4afd-9529-1f1a5de0b16d node DatanodeRegistration(127.0.0.1:40373, datanodeUuid=5966ade0-7b9a-4ced-87d7-5772caa42bfa, infoPort=46493, infoSecurePort=0, ipcPort=38205, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:39,099 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe55641d9713983ab with lease ID 0x86a82708ce3424a3: Processing first storage report for DS-b900acc2-d0bd-48ae-87cc-90a641d676f2 from datanode DatanodeRegistration(127.0.0.1:40373, datanodeUuid=5966ade0-7b9a-4ced-87d7-5772caa42bfa, infoPort=46493, infoSecurePort=0, ipcPort=38205, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279)
2024-11-22T17:35:39,099 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe55641d9713983ab with lease ID 0x86a82708ce3424a3: from storage DS-b900acc2-d0bd-48ae-87cc-90a641d676f2 node DatanodeRegistration(127.0.0.1:40373, datanodeUuid=5966ade0-7b9a-4ced-87d7-5772caa42bfa, infoPort=46493, infoSecurePort=0, ipcPort=38205, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:39,174 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,174 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,174 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,174 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,174 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:39,174 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:39,174 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:39,174 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta block BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:39,175 WARN  [PacketResponder: BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44863] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,175 WARN  [PacketResponder: BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:44863] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,176 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:37406 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:44863:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37406 dst: /127.0.0.1:44863
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:37656 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46859:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37656 dst: /127.0.0.1:46859
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:37654 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46859:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37654 dst: /127.0.0.1:46859
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1626607420_22 at /127.0.0.1:37622 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46859:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37622 dst: /127.0.0.1:46859
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_31597226_22 at /127.0.0.1:37680 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:46859:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37680 dst: /127.0.0.1:46859
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1626607420_22 at /127.0.0.1:37362 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:44863:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37362 dst: /127.0.0.1:44863
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_31597226_22 at /127.0.0.1:37430 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:44863:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37430 dst: /127.0.0.1:44863
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,177 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:37398 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:44863:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37398 dst: /127.0.0.1:44863
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:39,178 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3d88de8c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:39,179 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@36318c9a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:39,179 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:39,179 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56914f2c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:39,180 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3372b5da{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:39,180 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:39,180 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:39,180 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:39,180 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid 319bcf1b-27ff-4842-9143-cdb941ecdd5d) service to localhost/127.0.0.1:33455
2024-11-22T17:35:39,181 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:39,181 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:39,181 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:39,185 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,185 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,185 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta block BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,187 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,192 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dfaace0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:39,193 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@596eb72c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:39,193 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:39,193 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6425d7f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:39,193 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@172c9107{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:39,194 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:39,194 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:39,194 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:39,194 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid cdd6016f-30e9-4010-a8ca-29dba561881b) service to localhost/127.0.0.1:33455
2024-11-22T17:35:39,195 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data1/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:39,195 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data2/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:39,195 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., hostname=e61df7af61d2,34069,1732296925813, seqNum=2]
2024-11-22T17:35:39,200 ERROR [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24-prefix:e61df7af61d2,34069,1732296925813 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,201 WARN  [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24-prefix:e61df7af61d2,34069,1732296925813 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,201 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C34069%2C1732296925813:(num 1732296926480) roll requested
2024-11-22T17:35:39,201 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296939201
2024-11-22T17:35:39,207 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:39,207 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:39,207 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:39,207 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:39,207 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:39,207 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201
2024-11-22T17:35:39,208 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,208 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:39,209 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-22T17:35:39,209 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-22T17:35:39,209 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480
2024-11-22T17:35:39,212 WARN  [IPC Server handler 1 on default port 33455 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-22T17:35:39,213 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46493:46493),(127.0.0.1/127.0.0.1:43873:43873)]
2024-11-22T17:35:39,213 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:39,215 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 after 4ms
2024-11-22T17:35:39,942 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:41,159 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:41,214 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:41,215 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201
2024-11-22T17:35:41,216 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:41,216 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:41,217 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:57710 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:40373:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57710 dst: /127.0.0.1:40373
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:41,217 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:49712 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:38665:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49712 dst: /127.0.0.1:38665
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:41,283 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@312f47e7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:41,284 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20f3e03{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:41,284 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:41,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bbb3640{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:41,284 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75748ead{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:41,285 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:41,285 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid 5966ade0-7b9a-4ced-87d7-5772caa42bfa) service to localhost/127.0.0.1:33455
2024-11-22T17:35:41,285 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:41,285 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:41,286 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data9/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:41,286 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data10/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:41,286 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:41,942 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,159 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,215 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]
2024-11-22T17:35:43,215 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,215 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C34069%2C1732296925813:(num 1732296939201) roll requested
2024-11-22T17:35:43,216 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296943215
2024-11-22T17:35:43,217 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 after 4008ms
2024-11-22T17:35:43,220 WARN  [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,220 WARN  [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:43,220 WARN  [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741839_1021
2024-11-22T17:35:43,224 WARN  [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:43,228 WARN  [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,228 WARN  [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:43,228 WARN  [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741840_1022
2024-11-22T17:35:43,229 WARN  [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:43,230 WARN  [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:43,230 WARN  [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:43,230 WARN  [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741841_1023
2024-11-22T17:35:43,231 WARN  [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:43,234 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:43,235 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:43,235 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:43,235 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:43,235 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:43,235 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296943215
2024-11-22T17:35:43,236 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43873:43873),(127.0.0.1/127.0.0.1:38613:38613)]
2024-11-22T17:35:43,236 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:43,236 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201 is not closed yet, will try archiving it next time
2024-11-22T17:35:43,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38665 is added to blk_1073741838_1020 (size=2431)
2024-11-22T17:35:43,292 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-22T17:35:43,638 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:43,942 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,159 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,236 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,296 WARN  [ResponseProcessor for block BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,297 WARN  [DataStreamer for file /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296943215 block BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:45,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52740 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:38665:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52740 dst: /127.0.0.1:38665
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52858 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52858 dst: /127.0.0.1:35783
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,378 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@491335d6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:45,379 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74b1788d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:35:45,379 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:35:45,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24a7a164{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:35:45,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d7d08f9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:35:45,381 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:35:45,381 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid 462da758-2675-48dc-aa1d-756e2ebc551a) service to localhost/127.0.0.1:33455
2024-11-22T17:35:45,381 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:35:45,381 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:35:45,381 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data7/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:45,382 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data8/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:35:45,382 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:35:45,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:45,392 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:35:45,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/178bf755e16a478589304ee170403a8e is 1080, key is row0002/info:/1732296941288/Put/seqid=0
2024-11-22T17:35:45,419 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52876 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,419 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:45,419 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026
2024-11-22T17:35:45,419 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52876 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:45,420 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52876 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52876 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,420 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:45,421 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,422 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:45,422 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741844_1027
2024-11-22T17:35:45,422 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:45,424 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,424 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:45,424 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741845_1028
2024-11-22T17:35:45,425 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:45,428 WARN  [Thread-918 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46859
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:45,427 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52884 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029 to mirror 127.0.0.1:46859
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,428 WARN  [Thread-918 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:45,428 WARN  [Thread-918 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029
2024-11-22T17:35:45,428 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52884 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:45,428 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52884 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52884 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:45,429 WARN  [Thread-918 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:45,429 WARN  [IPC Server handler 2 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:45,429 WARN  [IPC Server handler 2 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:45,430 WARN  [IPC Server handler 2 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:45,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741847_1030 (size=10347)
2024-11-22T17:35:45,834 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/178bf755e16a478589304ee170403a8e
2024-11-22T17:35:45,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/178bf755e16a478589304ee170403a8e as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e
2024-11-22T17:35:45,853 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e, entries=5, sequenceid=11, filesize=10.1 K
2024-11-22T17:35:45,854 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 462ms, sequenceid=11, compaction requested=false
2024-11-22T17:35:45,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:45,943 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:46,020 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-22T17:35:46,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/00b8f2f84f0547d6832bc1a7cb4f47fd is 1080, key is row0007/info:/1732296945394/Put/seqid=0
2024-11-22T17:35:46,029 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:46,029 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52912 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:46,029 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:46,029 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031
2024-11-22T17:35:46,029 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52912 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:46,030 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52912 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52912 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:46,030 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:46,033 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:46,033 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52916 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:46,034 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:46,034 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032
2024-11-22T17:35:46,034 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52916 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032] {}] datanode.BlockReceiver(316): Block 1073741849 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:46,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52916 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741849_1032] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52916 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:46,035 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:46,036 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:46,036 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:46,036 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741850_1033
2024-11-22T17:35:46,037 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:46,038 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:46,038 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:46,038 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741851_1034
2024-11-22T17:35:46,039 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:46,039 WARN  [IPC Server handler 1 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:46,040 WARN  [IPC Server handler 1 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:46,040 WARN  [IPC Server handler 1 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:46,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741852_1035 (size=12506)
2024-11-22T17:35:46,444 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/00b8f2f84f0547d6832bc1a7cb4f47fd
2024-11-22T17:35:46,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/00b8f2f84f0547d6832bc1a7cb4f47fd as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd
2024-11-22T17:35:46,459 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd, entries=7, sequenceid=24, filesize=12.2 K
2024-11-22T17:35:46,460 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 441ms, sequenceid=24, compaction requested=false
2024-11-22T17:35:46,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-22T17:35:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:46,461 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd because midkey is the same as first or last row
2024-11-22T17:35:47,160 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,237 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]
2024-11-22T17:35:47,237 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,237 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C34069%2C1732296925813:(num 1732296943215) roll requested
2024-11-22T17:35:47,238 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296947237
2024-11-22T17:35:47,243 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,243 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:47,243 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741853_1036
2024-11-22T17:35:47,244 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:47,245 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,245 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:47,245 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741854_1037
2024-11-22T17:35:47,246 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:47,248 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40373
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,248 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52944 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038 to mirror 127.0.0.1:40373
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,249 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:47,249 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038
2024-11-22T17:35:47,249 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52944 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T17:35:47,249 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52944 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52944 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,249 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:47,252 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,252 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52952 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,252 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:47,252 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039
2024-11-22T17:35:47,252 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52952 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T17:35:47,252 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52952 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52952 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,253 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:47,254 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:47,254 WARN  [IPC Server handler 4 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:47,254 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:47,257 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:47,257 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:47,257 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:47,257 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:47,257 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:47,258 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296943215 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296947237
2024-11-22T17:35:47,259 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741842_1025 (size=25992)
2024-11-22T17:35:47,265 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38613:38613)]
2024-11-22T17:35:47,265 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:47,265 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296943215 is not closed yet, will try archiving it next time
2024-11-22T17:35:47,272 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296939201 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs/e61df7af61d2%2C34069%2C1732296925813.1732296939201
2024-11-22T17:35:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:47,443 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T17:35:47,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/310dcc05f0974ba8a3c9a8ee5208cd79 is 1079, key is tmprow/info:/1732296947442/Put/seqid=0
2024-11-22T17:35:47,450 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,451 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:47,451 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741858_1041
2024-11-22T17:35:47,451 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:47,454 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,454 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52984 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,454 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:47,454 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042
2024-11-22T17:35:47,454 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52984 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:47,454 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:52984 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52984 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,455 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:47,457 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46859
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,457 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53000 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043 to mirror 127.0.0.1:46859
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,458 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:47,458 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043
2024-11-22T17:35:47,458 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53000 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:47,458 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53000 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53000 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,458 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:47,461 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,461 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53002 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,461 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:47,461 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044
2024-11-22T17:35:47,462 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53002 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:47,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53002 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53002 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,462 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:47,463 WARN  [IPC Server handler 3 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:47,463 WARN  [IPC Server handler 3 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:47,463 WARN  [IPC Server handler 3 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:47,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741862_1045 (size=6027)
2024-11-22T17:35:47,661 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:47,868 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/310dcc05f0974ba8a3c9a8ee5208cd79
2024-11-22T17:35:47,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/310dcc05f0974ba8a3c9a8ee5208cd79 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79
2024-11-22T17:35:47,887 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79, entries=1, sequenceid=34, filesize=5.9 K
2024-11-22T17:35:47,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 445ms, sequenceid=34, compaction requested=true
2024-11-22T17:35:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-22T17:35:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd because midkey is the same as first or last row
2024-11-22T17:35:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b8a95d7bc5ab19d1e8c0d41584d89ec:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:35:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:35:47,889 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:35:47,890 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:35:47,890 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1541): 1b8a95d7bc5ab19d1e8c0d41584d89ec/info is initiating minor compaction (all files)
2024-11-22T17:35:47,891 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b8a95d7bc5ab19d1e8c0d41584d89ec/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:47,891 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79] into tmpdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp, totalSize=28.2 K
2024-11-22T17:35:47,891 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 178bf755e16a478589304ee170403a8e, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732296941288
2024-11-22T17:35:47,892 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b8f2f84f0547d6832bc1a7cb4f47fd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732296945394
2024-11-22T17:35:47,892 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310dcc05f0974ba8a3c9a8ee5208cd79, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732296947442
2024-11-22T17:35:47,907 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b8a95d7bc5ab19d1e8c0d41584d89ec#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:35:47,908 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/fd62bc5ab4f540438abe4baac0136242 is 1080, key is row0002/info:/1732296941288/Put/seqid=0
2024-11-22T17:35:47,910 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,910 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:47,910 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741863_1046
2024-11-22T17:35:47,911 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:47,912 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,913 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:47,913 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741864_1047
2024-11-22T17:35:47,913 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:47,915 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,915 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:47,915 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741865_1048
2024-11-22T17:35:47,915 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:47,918 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:47,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53054 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,918 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:47,918 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53054 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:47,918 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049
2024-11-22T17:35:47,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53054 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53054 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:47,919 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:47,920 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:47,920 WARN  [IPC Server handler 4 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:47,920 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:47,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741867_1050 (size=17994)
2024-11-22T17:35:47,943 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:48,339 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/fd62bc5ab4f540438abe4baac0136242 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242
2024-11-22T17:35:48,348 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b8a95d7bc5ab19d1e8c0d41584d89ec/info of 1b8a95d7bc5ab19d1e8c0d41584d89ec into fd62bc5ab4f540438abe4baac0136242(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:48,348 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., storeName=1b8a95d7bc5ab19d1e8c0d41584d89ec/info, priority=13, startTime=1732296947889; duration=0sec
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 because midkey is the same as first or last row
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 because midkey is the same as first or last row
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 because midkey is the same as first or last row
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:35:48,348 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b8a95d7bc5ab19d1e8c0d41584d89ec:info
2024-11-22T17:35:48,823 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3025a99[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741852_1035 to 127.0.0.1:40373 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,823 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@158372d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741847_1030 to 127.0.0.1:46859 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:48,865 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T17:35:48,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/243dd2c596204a8790e1dde4a202fb3c is 1079, key is tmprow/info:/1732296948863/Put/seqid=0
2024-11-22T17:35:48,871 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:48,872 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:48,872 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741868_1051
2024-11-22T17:35:48,872 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:48,878 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46859
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:48,878 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53072 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052 to mirror 127.0.0.1:46859
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,878 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:48,878 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052
2024-11-22T17:35:48,878 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53072 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:48,879 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53072 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53072 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,879 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:48,881 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:48,881 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:48,881 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741870_1053
2024-11-22T17:35:48,881 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:48,883 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:48,883 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53078 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,883 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:48,884 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054
2024-11-22T17:35:48,884 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53078 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:48,884 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53078 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53078 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:48,884 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:48,885 WARN  [IPC Server handler 3 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:48,885 WARN  [IPC Server handler 3 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:48,885 WARN  [IPC Server handler 3 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:48,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741872_1055 (size=6027)
2024-11-22T17:35:49,160 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,272 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]
2024-11-22T17:35:49,273 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,273 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C34069%2C1732296925813:(num 1732296947237) roll requested
2024-11-22T17:35:49,273 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296949273
2024-11-22T17:35:49,275 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,276 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:49,276 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741873_1056
2024-11-22T17:35:49,276 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:49,277 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,278 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:49,278 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741874_1057
2024-11-22T17:35:49,278 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:49,281 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53092 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,281 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:49,281 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058
2024-11-22T17:35:49,281 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53092 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T17:35:49,281 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53092 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53092 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,282 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:49,284 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44863
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:49,284 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53106 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059 to mirror 127.0.0.1:44863
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,284 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:49,284 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059
2024-11-22T17:35:49,284 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53106 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T17:35:49,284 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53106 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53106 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,285 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:49,285 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:49,285 WARN  [IPC Server handler 4 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:49,285 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:49,288 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:49,288 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:49,288 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:49,288 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:49,289 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:49,289 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296947237 with entries=14, filesize=12.92 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296949273
2024-11-22T17:35:49,289 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/243dd2c596204a8790e1dde4a202fb3c
2024-11-22T17:35:49,290 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38613:38613)]
2024-11-22T17:35:49,290 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:49,290 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296947237 is not closed yet, will try archiving it next time
2024-11-22T17:35:49,290 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296943215 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs/e61df7af61d2%2C34069%2C1732296925813.1732296943215
2024-11-22T17:35:49,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741857_1040 (size=13234)
2024-11-22T17:35:49,293 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 is not closed yet, will try archiving it next time
2024-11-22T17:35:49,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/243dd2c596204a8790e1dde4a202fb3c as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c
2024-11-22T17:35:49,305 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c, entries=1, sequenceid=45, filesize=5.9 K
2024-11-22T17:35:49,306 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 442ms, sequenceid=45, compaction requested=false
2024-11-22T17:35:49,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:49,306 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-22T17:35:49,306 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:49,306 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 because midkey is the same as first or last row
2024-11-22T17:35:49,813 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@158372d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741862_1045 to 127.0.0.1:44863 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,813 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3025a99[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741842_1025 to 127.0.0.1:40373 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:49,944 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:35:50,287 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T17:35:50,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/8bc4ad0ca4a5415ea056fe3d657b4020 is 1079, key is tmprow/info:/1732296950286/Put/seqid=0
2024-11-22T17:35:50,294 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,294 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK], DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:50,294 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741878_1061
2024-11-22T17:35:50,295 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:50,297 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40373
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53114 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062 to mirror 127.0.0.1:40373
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,297 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:50,297 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062
2024-11-22T17:35:50,297 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53114 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:50,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53114 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53114 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,298 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:50,299 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,299 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:50,299 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741880_1063
2024-11-22T17:35:50,300 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:50,301 WARN  [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,302 WARN  [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:50,302 WARN  [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741881_1064
2024-11-22T17:35:50,302 WARN  [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:50,303 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:50,303 WARN  [IPC Server handler 4 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:50,303 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:50,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741882_1065 (size=6027)
2024-11-22T17:35:50,707 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/8bc4ad0ca4a5415ea056fe3d657b4020
2024-11-22T17:35:50,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/8bc4ad0ca4a5415ea056fe3d657b4020 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020
2024-11-22T17:35:50,725 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020, entries=1, sequenceid=55, filesize=5.9 K
2024-11-22T17:35:50,726 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 439ms, sequenceid=55, compaction requested=true
2024-11-22T17:35:50,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:50,726 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-22T17:35:50,726 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:50,727 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 because midkey is the same as first or last row
2024-11-22T17:35:50,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b8a95d7bc5ab19d1e8c0d41584d89ec:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:35:50,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:35:50,727 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:35:50,728 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:35:50,728 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1541): 1b8a95d7bc5ab19d1e8c0d41584d89ec/info is initiating minor compaction (all files)
2024-11-22T17:35:50,728 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b8a95d7bc5ab19d1e8c0d41584d89ec/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:35:50,728 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020] into tmpdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp, totalSize=29.3 K
2024-11-22T17:35:50,729 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd62bc5ab4f540438abe4baac0136242, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732296941288
2024-11-22T17:35:50,729 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 243dd2c596204a8790e1dde4a202fb3c, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732296948863
2024-11-22T17:35:50,730 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bc4ad0ca4a5415ea056fe3d657b4020, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732296950286
2024-11-22T17:35:50,746 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b8a95d7bc5ab19d1e8c0d41584d89ec#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:35:50,747 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/7514c8ee5aaf4b7283c2261589d01ca3 is 1080, key is row0002/info:/1732296941288/Put/seqid=0
2024-11-22T17:35:50,749 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,749 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:50,749 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741883_1066
2024-11-22T17:35:50,749 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:50,752 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,752 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53130 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,752 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:50,752 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067
2024-11-22T17:35:50,752 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53130 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:50,752 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53130 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53130 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,753 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:50,754 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,754 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]) is bad.
2024-11-22T17:35:50,754 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741885_1068
2024-11-22T17:35:50,755 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44863,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]
2024-11-22T17:35:50,757 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46859
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:50,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53146 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069 to mirror 127.0.0.1:46859
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,757 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]) is bad.
2024-11-22T17:35:50,757 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069
2024-11-22T17:35:50,757 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53146 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:35:50,757 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:53146 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53146 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:50,758 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]
2024-11-22T17:35:50,759 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-22T17:35:50,759 WARN  [IPC Server handler 4 on default port 33455 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-22T17:35:50,759 WARN  [IPC Server handler 4 on default port 33455 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-22T17:35:50,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741887_1070 (size=18097)
2024-11-22T17:35:51,161 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:51,177 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/7514c8ee5aaf4b7283c2261589d01ca3 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3
2024-11-22T17:35:51,186 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b8a95d7bc5ab19d1e8c0d41584d89ec/info of 1b8a95d7bc5ab19d1e8c0d41584d89ec into 7514c8ee5aaf4b7283c2261589d01ca3(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:35:51,186 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:35:51,186 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., storeName=1b8a95d7bc5ab19d1e8c0d41584d89ec/info, priority=13, startTime=1732296950727; duration=0sec
2024-11-22T17:35:51,186 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-22T17:35:51,186 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:51,186 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 because midkey is the same as first or last row
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 because midkey is the same as first or last row
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 because midkey is the same as first or last row
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:35:51,187 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b8a95d7bc5ab19d1e8c0d41584d89ec:info
2024-11-22T17:35:51,290 WARN  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-22T17:35:51,290 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:51,313 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:35:51,318 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:35:51,319 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:35:51,319 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:35:51,319 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:35:51,320 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c7ddc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:35:51,320 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16760733{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:35:51,434 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@155d4993{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/java.io.tmpdir/jetty-localhost-42261-hadoop-hdfs-3_4_1-tests_jar-_-any-13573166484355626275/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:35:51,435 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17a639ae{HTTP/1.1, (http/1.1)}{localhost:42261}
2024-11-22T17:35:51,435 INFO  [Time-limited test {}] server.Server(415): Started @137473ms
2024-11-22T17:35:51,436 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:35:51,815 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@158372d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741872_1055 to 127.0.0.1:44863 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:51,815 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3025a99[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741867_1050 to 127.0.0.1:46859 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:51,944 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:51,962 WARN  [Thread-992 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:35:51,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6fc5cdfc4b8b948 with lease ID 0x86a82708ce3424a4: from storage DS-61e332e5-ddff-4342-ab2c-97e873a586bf node DatanodeRegistration(127.0.0.1:38051, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=39579, infoSecurePort=0, ipcPort=46143, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:35:51,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6fc5cdfc4b8b948 with lease ID 0x86a82708ce3424a4: from storage DS-074d536e-7101-4627-b352-f41f96358366 node DatanodeRegistration(127.0.0.1:38051, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=39579, infoSecurePort=0, ipcPort=46143, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:35:52,815 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3025a99[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741857_1040 to 127.0.0.1:40373 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:52,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741882_1065 (size=6027)
2024-11-22T17:35:53,161 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:53,291 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:53,945 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:54,814 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@158372d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35783, datanodeUuid=d36ebb12-48ad-4e68-88af-3b8a41c188a6, infoPort=38613, infoSecurePort=0, ipcPort=44519, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741887_1070 to 127.0.0.1:38665 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:55,161 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:55,291 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:55,605 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:35:55,945 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,191 ERROR [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData-prefix:e61df7af61d2,42763,1732296925630 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,191 WARN  [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData-prefix:e61df7af61d2,42763,1732296925630 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,191 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C42763%2C1732296925630:(num 1732296925963) roll requested
2024-11-22T17:35:56,192 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C42763%2C1732296925630.1732296956192
2024-11-22T17:35:56,195 WARN  [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,195 WARN  [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]) is bad.
2024-11-22T17:35:56,195 WARN  [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741888_1071
2024-11-22T17:35:56,196 WARN  [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40373,DS-8397671a-79e3-4afd-9529-1f1a5de0b16d,DISK]
2024-11-22T17:35:56,201 WARN  [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1072
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,201 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1626607420_22 at /127.0.0.1:55728 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:56,201 WARN  [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:35:56,201 WARN  [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072
2024-11-22T17:35:56,201 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1626607420_22 at /127.0.0.1:55728 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072] {}] datanode.BlockReceiver(316): Block 1073741889 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-22T17:35:56,201 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1626607420_22 at /127.0.0.1:55728 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741889_1072] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55728 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:35:56,202 WARN  [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:35:56,212 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:56,212 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:56,212 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:56,212 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:56,212 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:35:56,212 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296956192
2024-11-22T17:35:56,213 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,213 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:56,213 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963
2024-11-22T17:35:56,213 WARN  [IPC Server handler 4 on default port 33455 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 has not been closed. Lease recovery is in progress. RecoveryId = 1074 for block blk_1073741830_1006
2024-11-22T17:35:56,214 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 after 1ms
2024-11-22T17:35:56,217 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38613:38613),(127.0.0.1/127.0.0.1:39579:39579)]
2024-11-22T17:35:56,217 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 is not closed yet, will try archiving it next time
2024-11-22T17:35:57,162 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:57,292 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:59,162 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:35:59,292 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:00,215 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 after 4002ms
2024-11-22T17:36:01,163 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:01,293 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:01,986 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@224eb3c1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:46859,null,null])
java.net.ConnectException: Call From e61df7af61d2/172.17.0.2 to localhost:32977 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T17:36:01,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741833_1019 (size=455)
2024-11-22T17:36:02,235 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296926480 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs/e61df7af61d2%2C34069%2C1732296925813.1732296926480
2024-11-22T17:36:02,236 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296947237 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs/e61df7af61d2%2C34069%2C1732296925813.1732296947237
2024-11-22T17:36:03,163 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:03,293 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:03,967 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1326b031[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38051, datanodeUuid=319bcf1b-27ff-4842-9143-cdb941ecdd5d, infoPort=39579, infoSecurePort=0, ipcPort=46143, storageInfo=lv=-57;cid=testClusterID;nsid=460997743;c=1732296923279):Failed to transfer BP-1602666847-172.17.0.2-1732296923279:blk_1073741833_1019 to 127.0.0.1:40373 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:04,975 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.1732296964975
2024-11-22T17:36:04,979 WARN  [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1075
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:04,980 WARN  [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741891_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK], DatanodeInfoWithStorage[127.0.0.1:38051,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:04,980 WARN  [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741891_1075
2024-11-22T17:36:04,981 WARN  [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:04,985 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:04,986 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:04,986 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:04,986 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:04,986 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:04,986 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296949273 with entries=13, filesize=11.81 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296964975
2024-11-22T17:36:04,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38613:38613),(127.0.0.1/127.0.0.1:39579:39579)]
2024-11-22T17:36:04,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.1732296949273 is not closed yet, will try archiving it next time
2024-11-22T17:36:04,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741877_1060 (size=12100)
2024-11-22T17:36:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:36:04,996 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-22T17:36:05,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/d34ab9ab509a479a811dd211dbb27f90 is 1080, key is row0013/info:/1732296964989/Put/seqid=0
2024-11-22T17:36:05,005 WARN  [Thread-1030 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,005 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54232 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,005 WARN  [Thread-1030 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:05,005 WARN  [Thread-1030 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077
2024-11-22T17:36:05,005 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54232 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:36:05,005 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54232 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54232 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,006 WARN  [Thread-1030 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:05,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741894_1078 (size=9267)
2024-11-22T17:36:05,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741894_1078 (size=9267)
2024-11-22T17:36:05,031 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/d34ab9ab509a479a811dd211dbb27f90
2024-11-22T17:36:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/d34ab9ab509a479a811dd211dbb27f90 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90
2024-11-22T17:36:05,047 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90, entries=4, sequenceid=66, filesize=9.0 K
2024-11-22T17:36:05,048 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 53ms, sequenceid=66, compaction requested=false
2024-11-22T17:36:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:36:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.7 K, sizeToCheck=16.0 K
2024-11-22T17:36:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:36:05,048 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 because midkey is the same as first or last row
2024-11-22T17:36:05,163 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34069 {}] regionserver.HRegion(8855): Flush requested on 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:36:05,220 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1b8a95d7bc5ab19d1e8c0d41584d89ec 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-22T17:36:05,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/3f1baeb11f564a4da2a14c6f2f4f31e4 is 1080, key is row0016/info:/1732296964997/Put/seqid=0
2024-11-22T17:36:05,230 WARN  [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54256 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6]'}, localName='127.0.0.1:35783', datanodeUuid='d36ebb12-48ad-4e68-88af-3b8a41c188a6', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,230 WARN  [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35783,DS-7c760086-3f6c-492f-b610-ce53af0d973e,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:05,230 WARN  [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079
2024-11-22T17:36:05,230 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54256 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079] {}] datanode.BlockReceiver(316): Block 1073741895 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:36:05,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:54256 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741895_1079] {}] datanode.DataXceiver(331): 127.0.0.1:35783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54256 dst: /127.0.0.1:35783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,231 WARN  [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:05,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741896_1080 (size=13583)
2024-11-22T17:36:05,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741896_1080 (size=13583)
2024-11-22T17:36:05,237 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/3f1baeb11f564a4da2a14c6f2f4f31e4
2024-11-22T17:36:05,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/3f1baeb11f564a4da2a14c6f2f4f31e4 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4
2024-11-22T17:36:05,251 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4, entries=8, sequenceid=78, filesize=13.3 K
2024-11-22T17:36:05,252 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9682, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 1b8a95d7bc5ab19d1e8c0d41584d89ec in 33ms, sequenceid=78, compaction requested=true
2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 because midkey is the same as first or last row
2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1b8a95d7bc5ab19d1e8c0d41584d89ec:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:36:05,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:36:05,253 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:36:05,255 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:36:05,255 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1541): 1b8a95d7bc5ab19d1e8c0d41584d89ec/info is initiating minor compaction (all files)
2024-11-22T17:36:05,255 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1b8a95d7bc5ab19d1e8c0d41584d89ec/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,255 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4] into tmpdir=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp, totalSize=40.0 K
2024-11-22T17:36:05,256 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7514c8ee5aaf4b7283c2261589d01ca3, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732296941288
2024-11-22T17:36:05,256 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting d34ab9ab509a479a811dd211dbb27f90, keycount=4, bloomtype=ROW, size=9.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732296951100
2024-11-22T17:36:05,257 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1baeb11f564a4da2a14c6f2f4f31e4, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732296964997
2024-11-22T17:36:05,282 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1b8a95d7bc5ab19d1e8c0d41584d89ec#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:36:05,283 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/bee6034882ba44db9e2e12a62be2678c is 1080, key is row0002/info:/1732296941288/Put/seqid=0
2024-11-22T17:36:05,286 WARN  [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,286 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51494 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4]'}, localName='127.0.0.1:38051', datanodeUuid='319bcf1b-27ff-4842-9143-cdb941ecdd5d', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,286 WARN  [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38051,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:05,286 WARN  [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081
2024-11-22T17:36:05,286 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51494 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:36:05,286 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51494 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741897_1081] {}] datanode.DataXceiver(331): 127.0.0.1:38051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51494 dst: /127.0.0.1:38051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,287 WARN  [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:05,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741898_1082 (size=28989)
2024-11-22T17:36:05,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741898_1082 (size=28989)
2024-11-22T17:36:05,293 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-22T17:36:05,293 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,299 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/.tmp/info/bee6034882ba44db9e2e12a62be2678c as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/bee6034882ba44db9e2e12a62be2678c
2024-11-22T17:36:05,308 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1b8a95d7bc5ab19d1e8c0d41584d89ec/info of 1b8a95d7bc5ab19d1e8c0d41584d89ec into bee6034882ba44db9e2e12a62be2678c(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:36:05,308 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:

2024-11-22T17:36:05,308 INFO  [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., storeName=1b8a95d7bc5ab19d1e8c0d41584d89ec/info, priority=13, startTime=1732296965253; duration=0sec
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/bee6034882ba44db9e2e12a62be2678c because midkey is the same as first or last row
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/bee6034882ba44db9e2e12a62be2678c because midkey is the same as first or last row
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/bee6034882ba44db9e2e12a62be2678c because midkey is the same as first or last row
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:36:05,309 DEBUG [RS:0;e61df7af61d2:34069-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1b8a95d7bc5ab19d1e8c0d41584d89ec:info
2024-11-22T17:36:05,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:36:05,420 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:36:05,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:05,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:05,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:05,421 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:36:05,421 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:36:05,421 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=575691630, stopped=false
2024-11-22T17:36:05,421 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,42763,1732296925630
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:05,478 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:36:05,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:05,478 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:36:05,479 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:05,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:05,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:05,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:05,479 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,34069,1732296925813' *****
2024-11-22T17:36:05,479 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:36:05,479 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,41147,1732296927067' *****
2024-11-22T17:36:05,479 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:36:05,479 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:36:05,479 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:05,479 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:36:05,479 INFO  [RS:0;e61df7af61d2:34069 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:36:05,479 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:36:05,479 INFO  [RS:0;e61df7af61d2:34069 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:36:05,479 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:36:05,480 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,41147,1732296927067
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e61df7af61d2:41147.
2024-11-22T17:36:05,480 DEBUG [RS:1;e61df7af61d2:41147 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:05,480 DEBUG [RS:1;e61df7af61d2:41147 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:05,480 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,41147,1732296927067; all regions closed.
2024-11-22T17:36:05,480 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,34069,1732296925813
2024-11-22T17:36:05,480 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:36:05,480 INFO  [RS:0;e61df7af61d2:34069 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:34069.
2024-11-22T17:36:05,480 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b8a95d7bc5ab19d1e8c0d41584d89ec, disabling compactions & flushes
2024-11-22T17:36:05,480 DEBUG [RS:0;e61df7af61d2:34069 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:05,480 DEBUG [RS:0;e61df7af61d2:34069 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:05,480 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,480 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,481 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:36:05,481 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec. after waiting 0 ms
2024-11-22T17:36:05,481 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:36:05,481 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,481 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:36:05,481 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:36:05,481 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,481 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,481 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,481 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,481 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T17:36:05,481 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1325): Online Regions={1b8a95d7bc5ab19d1e8c0d41584d89ec=TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec., 1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:36:05,481 DEBUG [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1b8a95d7bc5ab19d1e8c0d41584d89ec
2024-11-22T17:36:05,481 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:36:05,481 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,481 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:36:05,482 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:36:05,482 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:36:05,481 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4] to archive
2024-11-22T17:36:05,482 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:36:05,482 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-22T17:36:05,482 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,482 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,482 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
2024-11-22T17:36:05,482 ERROR [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24-prefix:e61df7af61d2,34069,1732296925813.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,482 WARN  [FSHLog-0-hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24-prefix:e61df7af61d2,34069,1732296925813.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,483 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C34069%2C1732296925813.meta:.meta(num 1732296926879) roll requested
2024-11-22T17:36:05,483 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34069%2C1732296925813.meta.1732296965483.meta
2024-11-22T17:36:05,483 WARN  [IPC Server handler 1 on default port 33455 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 has not been closed. Lease recovery is in progress. RecoveryId = 1083 for block blk_1073741837_1013
2024-11-22T17:36:05,483 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:36:05,484 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 after 2ms
2024-11-22T17:36:05,486 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/178bf755e16a478589304ee170403a8e
2024-11-22T17:36:05,487 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/00b8f2f84f0547d6832bc1a7cb4f47fd
2024-11-22T17:36:05,489 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,489 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,489 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,489 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/fd62bc5ab4f540438abe4baac0136242
2024-11-22T17:36:05,489 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,490 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,490 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296965483.meta
2024-11-22T17:36:05,490 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,490 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46859,DS-92bbc77f-f18e-49e2-8039-5ee474d78577,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,490 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
2024-11-22T17:36:05,491 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/310dcc05f0974ba8a3c9a8ee5208cd79
2024-11-22T17:36:05,491 WARN  [IPC Server handler 0 on default port 33455 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta has not been closed. Lease recovery is in progress. RecoveryId = 1085 for block blk_1073741834_1010
2024-11-22T17:36:05,491 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta after 0ms
2024-11-22T17:36:05,492 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/243dd2c596204a8790e1dde4a202fb3c
2024-11-22T17:36:05,493 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38613:38613),(127.0.0.1/127.0.0.1:39579:39579)]
2024-11-22T17:36:05,493 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta is not closed yet, will try archiving it next time
2024-11-22T17:36:05,494 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/7514c8ee5aaf4b7283c2261589d01ca3
2024-11-22T17:36:05,497 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/8bc4ad0ca4a5415ea056fe3d657b4020
2024-11-22T17:36:05,498 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/d34ab9ab509a479a811dd211dbb27f90
2024-11-22T17:36:05,500 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/info/3f1baeb11f564a4da2a14c6f2f4f31e4
2024-11-22T17:36:05,500 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e61df7af61d2:42763 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T17:36:05,501 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [178bf755e16a478589304ee170403a8e=10347, 00b8f2f84f0547d6832bc1a7cb4f47fd=12506, fd62bc5ab4f540438abe4baac0136242=17994, 310dcc05f0974ba8a3c9a8ee5208cd79=6027, 243dd2c596204a8790e1dde4a202fb3c=6027, 7514c8ee5aaf4b7283c2261589d01ca3=18097, 8bc4ad0ca4a5415ea056fe3d657b4020=6027, d34ab9ab509a479a811dd211dbb27f90=9267, 3f1baeb11f564a4da2a14c6f2f4f31e4=13583]
2024-11-22T17:36:05,517 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/default/TestLogRolling-testLogRollOnDatanodeDeath/1b8a95d7bc5ab19d1e8c0d41584d89ec/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=1
2024-11-22T17:36:05,517 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,518 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b8a95d7bc5ab19d1e8c0d41584d89ec:
Waiting for close lock at 1732296965480Running coprocessor pre-close hooks at 1732296965480Disabling compacts and flushes for region at 1732296965480Disabling writes for close at 1732296965481 (+1 ms)Writing region close event to WAL at 1732296965512 (+31 ms)Running coprocessor post-close hooks at 1732296965517 (+5 ms)Closed at 1732296965517
2024-11-22T17:36:05,518 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec.
2024-11-22T17:36:05,532 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/info/e4cf9a8ea5a34ad5b1fb01d50c063b44 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732296927193.1b8a95d7bc5ab19d1e8c0d41584d89ec./info:regioninfo/1732296927994/Put/seqid=0
2024-11-22T17:36:05,535 WARN  [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51514 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4]'}, localName='127.0.0.1:38051', datanodeUuid='319bcf1b-27ff-4842-9143-cdb941ecdd5d', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,535 WARN  [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38051,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:05,535 WARN  [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086
2024-11-22T17:36:05,535 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51514 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086] {}] datanode.BlockReceiver(316): Block 1073741900 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:36:05,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51514 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741900_1086] {}] datanode.DataXceiver(331): 127.0.0.1:38051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51514 dst: /127.0.0.1:38051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,536 WARN  [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:05,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741901_1087 (size=7089)
2024-11-22T17:36:05,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741901_1087 (size=7089)
2024-11-22T17:36:05,541 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/info/e4cf9a8ea5a34ad5b1fb01d50c063b44
2024-11-22T17:36:05,567 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/ns/e0d9190618034dce911018e2425d8047 is 43, key is default/ns:d/1732296926957/Put/seqid=0
2024-11-22T17:36:05,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741902_1088 (size=5153)
2024-11-22T17:36:05,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741902_1088 (size=5153)
2024-11-22T17:36:05,573 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/ns/e0d9190618034dce911018e2425d8047
2024-11-22T17:36:05,599 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/table/c14d24406c63406caa94a61eca64c68b is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732296928010/Put/seqid=0
2024-11-22T17:36:05,602 WARN  [Thread-1074 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38665
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:05,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51556 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4]'}, localName='127.0.0.1:38051', datanodeUuid='319bcf1b-27ff-4842-9143-cdb941ecdd5d', xmitsInProgress=0}:Exception transferring block BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089 to mirror 127.0.0.1:38665
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,602 WARN  [Thread-1074 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38051,DS-61e332e5-ddff-4342-ab2c-97e873a586bf,DISK], DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]) is bad.
2024-11-22T17:36:05,602 WARN  [Thread-1074 {}] hdfs.DataStreamer(1850): Abandoning BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089
2024-11-22T17:36:05,602 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51556 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089] {}] datanode.BlockReceiver(316): Block 1073741903 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-22T17:36:05,602 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1574445872_22 at /127.0.0.1:51556 [Receiving block BP-1602666847-172.17.0.2-1732296923279:blk_1073741903_1089] {}] datanode.DataXceiver(331): 127.0.0.1:38051:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51556 dst: /127.0.0.1:38051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:05,603 WARN  [Thread-1074 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38665,DS-09a5c403-50f4-471b-aea9-64034f21cb30,DISK]
2024-11-22T17:36:05,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741904_1090 (size=5424)
2024-11-22T17:36:05,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741904_1090 (size=5424)
2024-11-22T17:36:05,616 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/table/c14d24406c63406caa94a61eca64c68b
2024-11-22T17:36:05,623 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/info/e4cf9a8ea5a34ad5b1fb01d50c063b44 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/info/e4cf9a8ea5a34ad5b1fb01d50c063b44
2024-11-22T17:36:05,631 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/info/e4cf9a8ea5a34ad5b1fb01d50c063b44, entries=10, sequenceid=11, filesize=6.9 K
2024-11-22T17:36:05,632 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/ns/e0d9190618034dce911018e2425d8047 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/ns/e0d9190618034dce911018e2425d8047
2024-11-22T17:36:05,640 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/ns/e0d9190618034dce911018e2425d8047, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T17:36:05,641 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/.tmp/table/c14d24406c63406caa94a61eca64c68b as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/table/c14d24406c63406caa94a61eca64c68b
2024-11-22T17:36:05,647 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/table/c14d24406c63406caa94a61eca64c68b, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T17:36:05,648 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 166ms, sequenceid=11, compaction requested=false
2024-11-22T17:36:05,654 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T17:36:05,654 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:36:05,655 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:05,655 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296965481Running coprocessor pre-close hooks at 1732296965481Disabling compacts and flushes for region at 1732296965481Disabling writes for close at 1732296965482 (+1 ms)Obtaining lock to block concurrent updates at 1732296965482Preparing flush snapshotting stores in 1588230740 at 1732296965482Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732296965483 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732296965493 (+10 ms)Flushing 1588230740/info: creating writer at 1732296965493Flushing 1588230740/info: appending metadata at 1732296965531 (+38 ms)Flushing 1588230740/info: closing flushed file at 1732296965531Flushing 1588230740/ns: creating writer at 1732296965548 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732296965566 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732296965566Flushing 1588230740/table: creating writer at 1732296965580 (+14 ms)Flushing 1588230740/table: appending metadata at 1732296965598 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732296965598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6366bcc8: reopening flushed file at 1732296965622 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e313058: reopening flushed file at 1732296965631 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56d16d2e: reopening flushed file at 1732296965640 (+9 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 166ms, sequenceid=11, compaction requested=false at 1732296965648 (+8 ms)Writing region close event to WAL at 1732296965650 (+2 ms)Running coprocessor post-close hooks at 1732296965654 (+4 ms)Closed at 1732296965654
2024-11-22T17:36:05,655 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:05,682 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,34069,1732296925813; all regions closed.
2024-11-22T17:36:05,682 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,682 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,682 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,682 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,683 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:05,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741899_1084 (size=825)
2024-11-22T17:36:05,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741899_1084 (size=825)
2024-11-22T17:36:06,158 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T17:36:06,159 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T17:36:06,346 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:06,399 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T17:36:06,400 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T17:36:06,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741835_1011 (size=393)
2024-11-22T17:36:06,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:36:06,972 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T17:36:06,972 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T17:36:07,160 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:07,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:36:07,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:36:09,485 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 after 4003ms
2024-11-22T17:36:09,492 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta after 4001ms
2024-11-22T17:36:09,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741877_1060 (size=12100)
2024-11-22T17:36:09,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:36:09,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:36:10,482 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-22T17:36:10,486 DEBUG [RS:1;e61df7af61d2:41147 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs
2024-11-22T17:36:10,486 INFO  [RS:1;e61df7af61d2:41147 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C41147%2C1732296927067:(num 1732296927294)
2024-11-22T17:36:10,486 DEBUG [RS:1;e61df7af61d2:41147 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:10,486 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:10,487 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:36:10,487 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:36:10,487 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:36:10,487 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:36:10,487 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:36:10,487 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:36:10,488 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:36:10,488 INFO  [RS:1;e61df7af61d2:41147 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41147
2024-11-22T17:36:10,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,540 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,540 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,540 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,541 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,41147,1732296927067
2024-11-22T17:36:10,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:36:10,551 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:36:10,561 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,561 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:10,562 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,41147,1732296927067]
2024-11-22T17:36:10,572 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,41147,1732296927067 already deleted, retry=false
2024-11-22T17:36:10,572 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,41147,1732296927067 expired; onlineServers=1
2024-11-22T17:36:10,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:10,662 INFO  [RS:1;e61df7af61d2:41147 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:36:10,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41147-0x101639d5d540002, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:10,662 INFO  [RS:1;e61df7af61d2:41147 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,41147,1732296927067; zookeeper connection closed.
2024-11-22T17:36:10,662 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e263fe8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e263fe8
2024-11-22T17:36:10,683 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-22T17:36:10,686 DEBUG [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs
2024-11-22T17:36:10,686 INFO  [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C34069%2C1732296925813.meta:.meta(num 1732296965483)
2024-11-22T17:36:10,687 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:10,687 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:10,687 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:10,687 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:10,687 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:10,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741892_1076 (size=16308)
2024-11-22T17:36:10,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741892_1076 (size=16308)
2024-11-22T17:36:10,693 DEBUG [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs
2024-11-22T17:36:10,693 INFO  [RS:0;e61df7af61d2:34069 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C34069%2C1732296925813:(num 1732296964975)
2024-11-22T17:36:10,693 DEBUG [RS:0;e61df7af61d2:34069 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:10,693 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:10,693 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:36:10,694 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-22T17:36:10,694 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:36:10,694 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:36:10,694 INFO  [RS:0;e61df7af61d2:34069 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34069
2024-11-22T17:36:10,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,34069,1732296925813
2024-11-22T17:36:10,703 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:36:10,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:36:10,793 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,34069,1732296925813]
2024-11-22T17:36:10,825 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,34069,1732296925813 already deleted, retry=false
2024-11-22T17:36:10,825 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,34069,1732296925813 expired; onlineServers=0
2024-11-22T17:36:10,825 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,42763,1732296925630' *****
2024-11-22T17:36:10,825 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:36:10,825 INFO  [M:0;e61df7af61d2:42763 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:36:10,825 INFO  [M:0;e61df7af61d2:42763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:36:10,825 DEBUG [M:0;e61df7af61d2:42763 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:36:10,825 DEBUG [M:0;e61df7af61d2:42763 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:36:10,825 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:36:10,825 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296926191 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296926191,5,FailOnTimeoutGroup]
2024-11-22T17:36:10,825 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296926191 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296926191,5,FailOnTimeoutGroup]
2024-11-22T17:36:10,826 INFO  [M:0;e61df7af61d2:42763 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:36:10,826 INFO  [M:0;e61df7af61d2:42763 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:36:10,826 DEBUG [M:0;e61df7af61d2:42763 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:36:10,826 INFO  [M:0;e61df7af61d2:42763 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:36:10,826 INFO  [M:0;e61df7af61d2:42763 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:36:10,827 INFO  [M:0;e61df7af61d2:42763 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:36:10,827 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:36:10,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:36:10,835 DEBUG [M:0;e61df7af61d2:42763 {}] zookeeper.ZKUtil(347): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:36:10,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:10,835 WARN  [M:0;e61df7af61d2:42763 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:36:10,836 INFO  [M:0;e61df7af61d2:42763 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/.lastflushedseqids
2024-11-22T17:36:10,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741905_1091 (size=130)
2024-11-22T17:36:10,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741905_1091 (size=130)
2024-11-22T17:36:10,849 INFO  [M:0;e61df7af61d2:42763 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:36:10,849 INFO  [M:0;e61df7af61d2:42763 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:36:10,849 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:36:10,849 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:10,849 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:10,849 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:36:10,849 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:10,849 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-22T17:36:10,866 DEBUG [M:0;e61df7af61d2:42763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aff20732f1ea4beab12946513a17f152 is 82, key is hbase:meta,,1/info:regioninfo/1732296926907/Put/seqid=0
2024-11-22T17:36:10,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741906_1092 (size=5672)
2024-11-22T17:36:10,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741906_1092 (size=5672)
2024-11-22T17:36:10,872 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aff20732f1ea4beab12946513a17f152
2024-11-22T17:36:10,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:10,893 INFO  [RS:0;e61df7af61d2:34069 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:36:10,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34069-0x101639d5d540001, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:10,893 INFO  [RS:0;e61df7af61d2:34069 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,34069,1732296925813; zookeeper connection closed.
2024-11-22T17:36:10,894 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c83cbb6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c83cbb6
2024-11-22T17:36:10,894 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-22T17:36:10,899 DEBUG [M:0;e61df7af61d2:42763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2deeaef103f46d4bb15fc201c4bcb07 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732296928016/Put/seqid=0
2024-11-22T17:36:10,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741907_1093 (size=6256)
2024-11-22T17:36:10,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741907_1093 (size=6256)
2024-11-22T17:36:10,906 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2deeaef103f46d4bb15fc201c4bcb07
2024-11-22T17:36:10,911 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2deeaef103f46d4bb15fc201c4bcb07
2024-11-22T17:36:10,929 DEBUG [M:0;e61df7af61d2:42763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/02669aee1b5e4c569df353c55f75c9b8 is 69, key is e61df7af61d2,34069,1732296925813/rs:state/1732296926308/Put/seqid=0
2024-11-22T17:36:10,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741908_1094 (size=5224)
2024-11-22T17:36:10,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741908_1094 (size=5224)
2024-11-22T17:36:10,935 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/02669aee1b5e4c569df353c55f75c9b8
2024-11-22T17:36:10,958 DEBUG [M:0;e61df7af61d2:42763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a205907f7724454a2cd3569e51f4926 is 52, key is load_balancer_on/state:d/1732296927053/Put/seqid=0
2024-11-22T17:36:10,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741909_1095 (size=5056)
2024-11-22T17:36:10,963 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741909_1095 (size=5056)
2024-11-22T17:36:10,963 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a205907f7724454a2cd3569e51f4926
2024-11-22T17:36:10,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:36:10,970 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/aff20732f1ea4beab12946513a17f152 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aff20732f1ea4beab12946513a17f152
2024-11-22T17:36:10,977 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/aff20732f1ea4beab12946513a17f152, entries=8, sequenceid=60, filesize=5.5 K
2024-11-22T17:36:10,978 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2deeaef103f46d4bb15fc201c4bcb07 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b2deeaef103f46d4bb15fc201c4bcb07
2024-11-22T17:36:10,984 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2deeaef103f46d4bb15fc201c4bcb07
2024-11-22T17:36:10,984 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b2deeaef103f46d4bb15fc201c4bcb07, entries=6, sequenceid=60, filesize=6.1 K
2024-11-22T17:36:10,985 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/02669aee1b5e4c569df353c55f75c9b8 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/02669aee1b5e4c569df353c55f75c9b8
2024-11-22T17:36:10,992 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/02669aee1b5e4c569df353c55f75c9b8, entries=2, sequenceid=60, filesize=5.1 K
2024-11-22T17:36:10,993 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/2a205907f7724454a2cd3569e51f4926 as hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2a205907f7724454a2cd3569e51f4926
2024-11-22T17:36:10,999 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/2a205907f7724454a2cd3569e51f4926, entries=1, sequenceid=60, filesize=4.9 K
2024-11-22T17:36:11,000 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false
2024-11-22T17:36:11,002 INFO  [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:11,002 DEBUG [M:0;e61df7af61d2:42763 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296970849Disabling compacts and flushes for region at 1732296970849Disabling writes for close at 1732296970849Obtaining lock to block concurrent updates at 1732296970849Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732296970849Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1732296970850 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732296970850Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732296970850Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732296970866 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732296970866Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732296970878 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732296970898 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732296970898Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732296970912 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732296970929 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732296970929Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732296970941 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732296970957 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732296970957Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f0adca: reopening flushed file at 1732296970969 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33768626: reopening flushed file at 1732296970977 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ebac5d8: reopening flushed file at 1732296970984 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6596f4a8: reopening flushed file at 1732296970992 (+8 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false at 1732296971000 (+8 ms)Writing region close event to WAL at 1732296971002 (+2 ms)Closed at 1732296971002
2024-11-22T17:36:11,002 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:11,003 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:11,003 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:11,003 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:11,003 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:11,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38051 is added to blk_1073741890_1073 (size=1045)
2024-11-22T17:36:11,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35783 is added to blk_1073741890_1073 (size=1045)
2024-11-22T17:36:11,063 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:36:11,081 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,082 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,082 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,083 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,083 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,089 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:11,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-22T17:36:11,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:36:11,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:36:11,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:36:11,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:11,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:11,990 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6d64680e {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:46859,null,null])
java.net.ConnectException: Call From e61df7af61d2/172.17.0.2 to localhost:32977 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T17:36:12,225 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/WALs/e61df7af61d2,42763,1732296925630/e61df7af61d2%2C42763%2C1732296925630.1732296925963 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/oldWALs/e61df7af61d2%2C42763%2C1732296925630.1732296925963
2024-11-22T17:36:12,231 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/MasterData/oldWALs/e61df7af61d2%2C42763%2C1732296925630.1732296925963 to hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/oldWALs/e61df7af61d2%2C42763%2C1732296925630.1732296925963$masterlocalwal$
2024-11-22T17:36:12,231 INFO  [M:0;e61df7af61d2:42763 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:36:12,231 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:36:12,231 INFO  [M:0;e61df7af61d2:42763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42763
2024-11-22T17:36:12,232 INFO  [M:0;e61df7af61d2:42763 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:36:12,351 INFO  [M:0;e61df7af61d2:42763 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:36:12,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:12,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42763-0x101639d5d540000, quorum=127.0.0.1:63933, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:12,387 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@155d4993{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:12,388 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17a639ae{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:12,388 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:12,389 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16760733{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:12,389 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51c7ddc0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:12,391 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:12,391 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:12,391 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid 319bcf1b-27ff-4842-9143-cdb941ecdd5d) service to localhost/127.0.0.1:33455
2024-11-22T17:36:12,391 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:12,391 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15e63548 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:46859,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:32977 , LocalHost:localPort e61df7af61d2/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-22T17:36:12,392 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15e63548 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1602666847-172.17.0.2-1732296923279:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:38051,null,null], DatanodeInfoWithStorage[127.0.0.1:46859,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1602666847-172.17.0.2-1732296923279
2024-11-22T17:36:12,392 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15e63548 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38051,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1602666847-172.17.0.2-1732296923279
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:12,392 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data3/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:12,392 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15e63548 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46859,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1602666847-172.17.0.2-1732296923279
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:12,393 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@15e63548 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:38051,null,null], DatanodeInfoWithStorage[127.0.0.1:46859,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1602666847-172.17.0.2-1732296923279:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:38051,null,null], DatanodeInfoWithStorage[127.0.0.1:46859,null,null]]
2024-11-22T17:36:12,393 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data4/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:12,393 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:12,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@57d162c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:12,407 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d72813f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:12,407 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:12,407 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7857b965{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:12,407 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41b3c9b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:12,408 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:12,408 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:12,408 WARN  [BP-1602666847-172.17.0.2-1732296923279 heartbeating to localhost/127.0.0.1:33455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1602666847-172.17.0.2-1732296923279 (Datanode Uuid d36ebb12-48ad-4e68-88af-3b8a41c188a6) service to localhost/127.0.0.1:33455
2024-11-22T17:36:12,408 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:12,409 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data5/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:12,409 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/cluster_5a8944c6-3c2d-da9f-ce62-9c5cdb4be4a5/data/data6/current/BP-1602666847-172.17.0.2-1732296923279 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:12,409 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:12,415 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d90daca{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:36:12,416 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16ad4404{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:12,416 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:12,416 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ff0d030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:12,416 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37564f36{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:12,425 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:36:12,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:36:12,467 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 79)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-14-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:33455 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:33455 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:33455
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33455
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fd418befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37617
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fd418befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37617
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-14-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33455
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:33455
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33455
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33455
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:33455 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:33455 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33455
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-15-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 225) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1936 (was 2844)
2024-11-22T17:36:12,475 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=1935
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.log.dir so I do NOT create it in target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b488fe8-384c-dcde-7120-4f972fd38fee/hadoop.tmp.dir so I do NOT create it in target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f, deleteOnExit=true
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/test.cache.data in system properties and HBase conf
2024-11-22T17:36:12,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:36:12,477 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:36:12,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:36:12,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:36:12,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:12,492 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:36:12,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:12,884 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:12,889 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:12,890 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:12,890 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:12,890 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:12,891 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:12,892 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b850ed0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:12,892 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7115368f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:13,003 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@113238fc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-45949-hadoop-hdfs-3_4_1-tests_jar-_-any-13138644814785014800/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:36:13,004 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c79a3ba{HTTP/1.1, (http/1.1)}{localhost:45949}
2024-11-22T17:36:13,004 INFO  [Time-limited test {}] server.Server(415): Started @159041ms
2024-11-22T17:36:13,016 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:36:13,245 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:13,249 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:13,250 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:13,250 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:13,250 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:13,251 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d0d4ae6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:13,251 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28abb6ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:13,357 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64527816{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-40695-hadoop-hdfs-3_4_1-tests_jar-_-any-7788326347438218355/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:13,357 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1799acad{HTTP/1.1, (http/1.1)}{localhost:40695}
2024-11-22T17:36:13,358 INFO  [Time-limited test {}] server.Server(415): Started @159395ms
2024-11-22T17:36:13,359 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:13,415 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:13,420 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:13,424 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:13,425 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:13,425 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:36:13,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5479887a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:13,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f93d049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:13,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:13,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:13,534 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b55dda{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-34853-hadoop-hdfs-3_4_1-tests_jar-_-any-16813501281495050985/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:13,534 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f0797d2{HTTP/1.1, (http/1.1)}{localhost:34853}
2024-11-22T17:36:13,534 INFO  [Time-limited test {}] server.Server(415): Started @159572ms
2024-11-22T17:36:13,536 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:14,349 WARN  [Thread-1216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data1/current/BP-806922106-172.17.0.2-1732296972497/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:14,356 WARN  [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data2/current/BP-806922106-172.17.0.2-1732296972497/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:14,384 WARN  [Thread-1180 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:14,386 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc9ebd48653e427e with lease ID 0x499f6377f563214: Processing first storage report for DS-46de46a3-2c67-46a6-a707-8ab286489e9f from datanode DatanodeRegistration(127.0.0.1:41033, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=46279, infoSecurePort=0, ipcPort=41167, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497)
2024-11-22T17:36:14,386 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc9ebd48653e427e with lease ID 0x499f6377f563214: from storage DS-46de46a3-2c67-46a6-a707-8ab286489e9f node DatanodeRegistration(127.0.0.1:41033, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=46279, infoSecurePort=0, ipcPort=41167, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:14,387 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc9ebd48653e427e with lease ID 0x499f6377f563214: Processing first storage report for DS-0ea4a2e9-d250-4493-b3af-88e30981425d from datanode DatanodeRegistration(127.0.0.1:41033, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=46279, infoSecurePort=0, ipcPort=41167, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497)
2024-11-22T17:36:14,387 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc9ebd48653e427e with lease ID 0x499f6377f563214: from storage DS-0ea4a2e9-d250-4493-b3af-88e30981425d node DatanodeRegistration(127.0.0.1:41033, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=46279, infoSecurePort=0, ipcPort=41167, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:14,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:14,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:14,561 WARN  [Thread-1227 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data3/current/BP-806922106-172.17.0.2-1732296972497/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:14,561 WARN  [Thread-1228 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data4/current/BP-806922106-172.17.0.2-1732296972497/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:14,580 WARN  [Thread-1203 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:14,582 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbf2920440a4105a7 with lease ID 0x499f6377f563215: Processing first storage report for DS-56c7e442-b398-4304-932f-9a60320019de from datanode DatanodeRegistration(127.0.0.1:33869, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=37765, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497)
2024-11-22T17:36:14,583 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbf2920440a4105a7 with lease ID 0x499f6377f563215: from storage DS-56c7e442-b398-4304-932f-9a60320019de node DatanodeRegistration(127.0.0.1:33869, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=37765, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:14,583 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbf2920440a4105a7 with lease ID 0x499f6377f563215: Processing first storage report for DS-a5981da5-f013-4351-990a-2a585d6f9fc1 from datanode DatanodeRegistration(127.0.0.1:33869, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=37765, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497)
2024-11-22T17:36:14,583 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbf2920440a4105a7 with lease ID 0x499f6377f563215: from storage DS-a5981da5-f013-4351-990a-2a585d6f9fc1 node DatanodeRegistration(127.0.0.1:33869, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=37765, infoSecurePort=0, ipcPort=38147, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:14,672 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566
2024-11-22T17:36:14,676 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/zookeeper_0, clientPort=62961, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:36:14,677 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62961
2024-11-22T17:36:14,678 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,679 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:36:14,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:36:14,691 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153 with version=8
2024-11-22T17:36:14,691 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:36:14,693 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:36:14,693 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:36:14,694 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:36:14,695 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45723
2024-11-22T17:36:14,697 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45723 connecting to ZooKeeper ensemble=127.0.0.1:62961
2024-11-22T17:36:14,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:457230x0, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:36:14,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45723-0x101639e1cff0000 connected
2024-11-22T17:36:14,846 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,848 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:14,850 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153, hbase.cluster.distributed=false
2024-11-22T17:36:14,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:36:14,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45723
2024-11-22T17:36:14,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45723
2024-11-22T17:36:14,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45723
2024-11-22T17:36:14,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45723
2024-11-22T17:36:14,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45723
2024-11-22T17:36:14,869 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:36:14,869 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:36:14,870 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:36:14,871 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44007
2024-11-22T17:36:14,872 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44007 connecting to ZooKeeper ensemble=127.0.0.1:62961
2024-11-22T17:36:14,873 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,875 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440070x0, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:36:14,888 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:440070x0, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:14,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44007-0x101639e1cff0001 connected
2024-11-22T17:36:14,888 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:36:14,889 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:36:14,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:36:14,891 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:36:14,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44007
2024-11-22T17:36:14,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44007
2024-11-22T17:36:14,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44007
2024-11-22T17:36:14,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44007
2024-11-22T17:36:14,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44007
2024-11-22T17:36:14,913 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:45723
2024-11-22T17:36:14,914 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,45723,1732296974693
2024-11-22T17:36:14,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:14,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:14,924 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,45723,1732296974693
2024-11-22T17:36:14,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:14,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:36:14,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:14,935 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:36:14,935 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,45723,1732296974693 from backup master directory
2024-11-22T17:36:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,45723,1732296974693
2024-11-22T17:36:14,945 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:36:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:14,945 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,45723,1732296974693
2024-11-22T17:36:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:14,951 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/hbase.id] with ID: d9261edf-4f59-44c9-a7e0-78b265553292
2024-11-22T17:36:14,951 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/.tmp/hbase.id
2024-11-22T17:36:14,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:36:14,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:36:14,958 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/.tmp/hbase.id]:[hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/hbase.id]
2024-11-22T17:36:14,972 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:14,972 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:36:14,973 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-22T17:36:14,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:14,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:14,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:36:14,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:36:14,995 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:36:14,996 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:36:14,996 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:15,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:36:15,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:36:15,017 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store
2024-11-22T17:36:15,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:36:15,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:36:15,031 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:15,031 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:36:15,031 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:15,031 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:15,032 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:36:15,032 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:15,032 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:15,032 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732296975031Disabling compacts and flushes for region at 1732296975031Disabling writes for close at 1732296975032 (+1 ms)Writing region close event to WAL at 1732296975032Closed at 1732296975032
2024-11-22T17:36:15,032 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/.initializing
2024-11-22T17:36:15,033 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693
2024-11-22T17:36:15,039 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C45723%2C1732296974693, suffix=, logDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693, archiveDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/oldWALs, maxLogs=10
2024-11-22T17:36:15,040 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C45723%2C1732296974693.1732296975040
2024-11-22T17:36:15,046 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040
2024-11-22T17:36:15,052 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46279:46279),(127.0.0.1/127.0.0.1:37765:37765)]
2024-11-22T17:36:15,056 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:15,057 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:15,057 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,057 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,059 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:15,060 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:36:15,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,061 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,061 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:15,063 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:36:15,063 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,063 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:15,064 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:15,065 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:36:15,065 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,066 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:15,066 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:15,067 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:36:15,067 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,068 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:15,068 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,069 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,069 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,071 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,071 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,071 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:36:15,072 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:15,075 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:15,076 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727027, jitterRate=-0.07553817331790924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:36:15,077 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732296975057Initializing all the Stores at 1732296975058 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975058Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296975058Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296975058Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296975059 (+1 ms)Cleaning up temporary data from old regions at 1732296975071 (+12 ms)Region opened successfully at 1732296975077 (+6 ms)
2024-11-22T17:36:15,077 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:36:15,080 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70fb2b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:36:15,081 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:36:15,082 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:36:15,082 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:36:15,082 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:36:15,082 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:36:15,083 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:36:15,083 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:36:15,085 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:36:15,086 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:36:15,098 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:36:15,099 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:36:15,100 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:36:15,108 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:36:15,109 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:36:15,110 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:36:15,119 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:36:15,120 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:36:15,129 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:36:15,132 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:36:15,140 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:36:15,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:15,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:15,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,151 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,45723,1732296974693, sessionid=0x101639e1cff0000, setting cluster-up flag (Was=false)
2024-11-22T17:36:15,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,203 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:36:15,205 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,45723,1732296974693
2024-11-22T17:36:15,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,261 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:36:15,262 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,45723,1732296974693
2024-11-22T17:36:15,264 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:36:15,266 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:15,267 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:36:15,267 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:36:15,267 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,45723,1732296974693
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:36:15,269 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:15,269 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:15,269 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:15,269 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:15,270 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:36:15,270 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,270 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:36:15,270 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732297005271
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:36:15,271 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:36:15,272 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:15,272 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:36:15,272 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:36:15,272 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:36:15,272 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:36:15,272 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:36:15,273 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:36:15,273 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296975273,5,FailOnTimeoutGroup]
2024-11-22T17:36:15,273 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296975273,5,FailOnTimeoutGroup]
2024-11-22T17:36:15,273 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,273 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:36:15,273 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,273 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,274 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,274 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:36:15,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:36:15,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:36:15,284 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:36:15,284 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153
2024-11-22T17:36:15,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:36:15,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:36:15,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:15,294 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:36:15,296 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:36:15,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,296 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,297 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:36:15,298 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:36:15,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,299 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,299 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:36:15,300 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:36:15,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,301 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,301 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:36:15,302 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(746): ClusterId : d9261edf-4f59-44c9-a7e0-78b265553292
2024-11-22T17:36:15,302 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:36:15,303 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:36:15,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,303 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:36:15,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740
2024-11-22T17:36:15,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740
2024-11-22T17:36:15,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:36:15,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:36:15,306 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:36:15,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:36:15,309 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:15,310 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=747868, jitterRate=-0.04903765022754669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:36:15,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732296975293Initializing all the Stores at 1732296975294 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975294Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975294Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296975294Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975294Cleaning up temporary data from old regions at 1732296975306 (+12 ms)Region opened successfully at 1732296975310 (+4 ms)
2024-11-22T17:36:15,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:36:15,311 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:36:15,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:36:15,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:36:15,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:36:15,311 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:15,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732296975311Disabling compacts and flushes for region at 1732296975311Disabling writes for close at 1732296975311Writing region close event to WAL at 1732296975311Closed at 1732296975311
2024-11-22T17:36:15,312 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:15,312 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:36:15,313 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:36:15,314 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:36:15,314 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:36:15,314 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:36:15,315 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:36:15,325 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:36:15,325 DEBUG [RS:0;e61df7af61d2:44007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ac4b3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:36:15,336 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:44007
2024-11-22T17:36:15,336 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:36:15,336 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:36:15,336 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:36:15,337 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,45723,1732296974693 with port=44007, startcode=1732296974869
2024-11-22T17:36:15,337 DEBUG [RS:0;e61df7af61d2:44007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:36:15,339 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:36:15,339 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45723 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,339 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45723 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,341 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153
2024-11-22T17:36:15,341 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46471
2024-11-22T17:36:15,341 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:36:15,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:36:15,393 DEBUG [RS:0;e61df7af61d2:44007 {}] zookeeper.ZKUtil(111): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,393 WARN  [RS:0;e61df7af61d2:44007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:36:15,393 INFO  [RS:0;e61df7af61d2:44007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:15,394 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,394 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,44007,1732296974869]
2024-11-22T17:36:15,396 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:36:15,398 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:36:15,398 INFO  [RS:0;e61df7af61d2:44007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:36:15,398 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,398 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:36:15,399 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:36:15,399 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,399 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:36:15,400 DEBUG [RS:0;e61df7af61d2:44007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,401 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,44007,1732296974869-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:36:15,414 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:36:15,415 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,44007,1732296974869-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,415 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,415 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.Replication(171): e61df7af61d2,44007,1732296974869 started
2024-11-22T17:36:15,428 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:15,428 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,44007,1732296974869, RpcServer on e61df7af61d2/172.17.0.2:44007, sessionid=0x101639e1cff0001
2024-11-22T17:36:15,428 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:36:15,428 DEBUG [RS:0;e61df7af61d2:44007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,428 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,44007,1732296974869'
2024-11-22T17:36:15,428 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,44007,1732296974869'
2024-11-22T17:36:15,429 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:36:15,430 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:36:15,430 DEBUG [RS:0;e61df7af61d2:44007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:36:15,430 INFO  [RS:0;e61df7af61d2:44007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:36:15,430 INFO  [RS:0;e61df7af61d2:44007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:36:15,466 WARN  [e61df7af61d2:45723 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:36:15,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:15,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:15,532 INFO  [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C44007%2C1732296974869, suffix=, logDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869, archiveDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs, maxLogs=32
2024-11-22T17:36:15,533 INFO  [RS:0;e61df7af61d2:44007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:15,539 INFO  [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:15,540 DEBUG [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37765:37765),(127.0.0.1/127.0.0.1:46279:46279)]
2024-11-22T17:36:15,716 DEBUG [e61df7af61d2:45723 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:36:15,717 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,719 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,44007,1732296974869, state=OPENING
2024-11-22T17:36:15,745 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:36:15,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:15,756 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:36:15,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:15,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:15,756 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,44007,1732296974869}]
2024-11-22T17:36:15,911 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:36:15,913 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:36:15,918 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:36:15,918 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:15,921 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C44007%2C1732296974869.meta, suffix=.meta, logDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869, archiveDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs, maxLogs=32
2024-11-22T17:36:15,921 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta
2024-11-22T17:36:15,929 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta
2024-11-22T17:36:15,930 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46279:46279),(127.0.0.1/127.0.0.1:37765:37765)]
2024-11-22T17:36:15,931 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:15,931 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:36:15,931 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:36:15,931 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:36:15,932 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:36:15,932 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:15,932 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:36:15,932 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:36:15,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:36:15,935 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:36:15,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,935 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:36:15,936 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:36:15,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,937 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:36:15,938 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:36:15,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:36:15,940 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:36:15,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:15,940 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:15,940 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:36:15,941 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740
2024-11-22T17:36:15,943 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740
2024-11-22T17:36:15,944 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:36:15,944 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:36:15,945 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:36:15,946 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:36:15,947 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877045, jitterRate=0.1152208000421524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:36:15,947 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:36:15,948 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732296975932Writing region info on filesystem at 1732296975932Initializing all the Stores at 1732296975933 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975933Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975934 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296975934Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732296975934Cleaning up temporary data from old regions at 1732296975944 (+10 ms)Running coprocessor post-open hooks at 1732296975947 (+3 ms)Region opened successfully at 1732296975948 (+1 ms)
2024-11-22T17:36:15,949 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732296975910
2024-11-22T17:36:15,951 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:36:15,951 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:36:15,952 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:15,953 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,44007,1732296974869, state=OPEN
2024-11-22T17:36:16,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:36:16,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:36:16,015 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:16,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:16,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:16,019 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:36:16,019 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,44007,1732296974869 in 259 msec
2024-11-22T17:36:16,022 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:36:16,022 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 706 msec
2024-11-22T17:36:16,024 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:16,024 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:36:16,025 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:36:16,026 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,44007,1732296974869, seqNum=-1]
2024-11-22T17:36:16,026 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:36:16,027 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:36:16,033 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 766 msec
2024-11-22T17:36:16,034 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732296976033, completionTime=-1
2024-11-22T17:36:16,034 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:36:16,034 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732297036036
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297096036
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,036 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:45723, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,037 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,037 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,039 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.096sec
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:36:16,041 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:36:16,044 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:36:16,044 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:36:16,044 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,45723,1732296974693-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:16,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ca2be4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:16,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,45723,-1 for getting cluster id
2024-11-22T17:36:16,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:36:16,105 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9261edf-4f59-44c9-a7e0-78b265553292'
2024-11-22T17:36:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:36:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9261edf-4f59-44c9-a7e0-78b265553292"

2024-11-22T17:36:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c36c56b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,45723,-1]
2024-11-22T17:36:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:36:16,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:16,109 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:36:16,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9d252f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:16,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:36:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,44007,1732296974869, seqNum=-1]
2024-11-22T17:36:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:36:16,114 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:36:16,117 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,45723,1732296974693
2024-11-22T17:36:16,117 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:16,120 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:36:16,120 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-22T17:36:16,120 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-22T17:36:16,121 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T17:36:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,45723,1732296974693
2024-11-22T17:36:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12342082
2024-11-22T17:36:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T17:36:16,124 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T17:36:16,125 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T17:36:16,125 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T17:36:16,125 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:36:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T17:36:16,129 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T17:36:16,130 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:16,130 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-22T17:36:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:36:16,132 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T17:36:16,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741835_1011 (size=395)
2024-11-22T17:36:16,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741835_1011 (size=395)
2024-11-22T17:36:16,143 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e65354aba8acff3f70c7d54b2ed437d4, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153
2024-11-22T17:36:16,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41033 is added to blk_1073741836_1012 (size=78)
2024-11-22T17:36:16,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33869 is added to blk_1073741836_1012 (size=78)
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing e65354aba8acff3f70c7d54b2ed437d4, disabling compactions & flushes
2024-11-22T17:36:16,151 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4. after waiting 0 ms
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,151 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,151 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for e65354aba8acff3f70c7d54b2ed437d4:
Waiting for close lock at 1732296976151Disabling compacts and flushes for region at 1732296976151Disabling writes for close at 1732296976151Writing region close event to WAL at 1732296976151Closed at 1732296976151
2024-11-22T17:36:16,153 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T17:36:16,153 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732296976153"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732296976153"}]},"ts":"1732296976153"}
2024-11-22T17:36:16,156 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T17:36:16,157 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T17:36:16,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296976157"}]},"ts":"1732296976157"}
2024-11-22T17:36:16,159 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-22T17:36:16,160 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=e65354aba8acff3f70c7d54b2ed437d4, ASSIGN}]
2024-11-22T17:36:16,161 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=e65354aba8acff3f70c7d54b2ed437d4, ASSIGN
2024-11-22T17:36:16,162 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=e65354aba8acff3f70c7d54b2ed437d4, ASSIGN; state=OFFLINE, location=e61df7af61d2,44007,1732296974869; forceNewPlan=false, retain=false
2024-11-22T17:36:16,313 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e65354aba8acff3f70c7d54b2ed437d4, regionState=OPENING, regionLocation=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:16,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=e65354aba8acff3f70c7d54b2ed437d4, ASSIGN because future has completed
2024-11-22T17:36:16,317 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e65354aba8acff3f70c7d54b2ed437d4, server=e61df7af61d2,44007,1732296974869}]
2024-11-22T17:36:16,476 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,477 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e65354aba8acff3f70c7d54b2ed437d4, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:16,477 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,477 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:16,478 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,478 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,480 INFO  [StoreOpener-e65354aba8acff3f70c7d54b2ed437d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e65354aba8acff3f70c7d54b2ed437d4 
2024-11-22T17:36:16,482 INFO  [StoreOpener-e65354aba8acff3f70c7d54b2ed437d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e65354aba8acff3f70c7d54b2ed437d4 columnFamilyName info
2024-11-22T17:36:16,482 DEBUG [StoreOpener-e65354aba8acff3f70c7d54b2ed437d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:16,482 INFO  [StoreOpener-e65354aba8acff3f70c7d54b2ed437d4-1 {}] regionserver.HStore(327): Store=e65354aba8acff3f70c7d54b2ed437d4/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:16,482 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,483 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,484 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,484 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,484 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,486 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,488 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:16,489 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e65354aba8acff3f70c7d54b2ed437d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=753495, jitterRate=-0.041882097721099854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:36:16,489 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:16,490 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e65354aba8acff3f70c7d54b2ed437d4:
Running coprocessor pre-open hook at 1732296976478Writing region info on filesystem at 1732296976478Initializing all the Stores at 1732296976479 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732296976479Cleaning up temporary data from old regions at 1732296976484 (+5 ms)Running coprocessor post-open hooks at 1732296976489 (+5 ms)Region opened successfully at 1732296976490 (+1 ms)
2024-11-22T17:36:16,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:16,491 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4., pid=6, masterSystemTime=1732296976470
2024-11-22T17:36:16,494 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,494 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:16,495 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e65354aba8acff3f70c7d54b2ed437d4, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,44007,1732296974869
2024-11-22T17:36:16,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:16,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e65354aba8acff3f70c7d54b2ed437d4, server=e61df7af61d2,44007,1732296974869 because future has completed
2024-11-22T17:36:16,503 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T17:36:16,503 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e65354aba8acff3f70c7d54b2ed437d4, server=e61df7af61d2,44007,1732296974869 in 183 msec
2024-11-22T17:36:16,506 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T17:36:16,506 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=e65354aba8acff3f70c7d54b2ed437d4, ASSIGN in 343 msec
2024-11-22T17:36:16,507 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T17:36:16,507 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732296976507"}]},"ts":"1732296976507"}
2024-11-22T17:36:16,509 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-22T17:36:16,510 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T17:36:16,513 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 385 msec
2024-11-22T17:36:16,709 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:36:16,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,730 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,731 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,732 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,737 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:16,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:17,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:17,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:18,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:18,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:19,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:19,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:20,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:20,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:21,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:36:21,205 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T17:36:21,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T17:36:21,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-22T17:36:21,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:36:21,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T17:36:21,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:36:21,207 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T17:36:21,397 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T17:36:21,397 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-22T17:36:21,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:21,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:21,980 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:36:22,016 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,016 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,017 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,026 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,027 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,031 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:22,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:22,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:23,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:23,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:24,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:24,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:25,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:25,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45723 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:36:26,204 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-22T17:36:26,204 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-22T17:36:26,209 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T17:36:26,209 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4., hostname=e61df7af61d2,44007,1732296974869, seqNum=2]
2024-11-22T17:36:26,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:26,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:27,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:27,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:28,217 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:28,217 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,218 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK], DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]) is bad.
2024-11-22T17:36:28,218 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:48904 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33869:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48904 dst: /127.0.0.1:33869
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,218 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,219 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK], DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]) is bad.
2024-11-22T17:36:28,219 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,219 WARN  [PacketResponder: BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33869] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,219 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK], DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33869,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]) is bad.
2024-11-22T17:36:28,219 WARN  [PacketResponder: BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33869] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,220 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:39986 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39986 dst: /127.0.0.1:41033
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,220 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-443229131_22 at /127.0.0.1:39952 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39952 dst: /127.0.0.1:41033
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,220 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-443229131_22 at /127.0.0.1:48878 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33869:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48878 dst: /127.0.0.1:33869
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,220 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:40002 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40002 dst: /127.0.0.1:41033
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,220 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:48918 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33869:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48918 dst: /127.0.0.1:33869
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,222 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b55dda{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:28,223 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f0797d2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:28,223 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:28,223 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f93d049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:28,223 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5479887a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:28,224 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:28,224 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid d9a2ad13-e794-44fe-bc3e-7b72309884b1) service to localhost/127.0.0.1:46471
2024-11-22T17:36:28,224 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:28,224 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:28,225 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data3/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:28,225 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data4/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:28,225 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:28,282 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:28,285 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:28,286 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:28,286 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:28,286 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:28,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19409378{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:28,287 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cda8c7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:28,391 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@68a7eef1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-34221-hadoop-hdfs-3_4_1-tests_jar-_-any-14686016357625268183/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:28,391 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a86c024{HTTP/1.1, (http/1.1)}{localhost:34221}
2024-11-22T17:36:28,392 INFO  [Time-limited test {}] server.Server(415): Started @174429ms
2024-11-22T17:36:28,393 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:28,449 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,449 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,449 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:28,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-443229131_22 at /127.0.0.1:34570 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34570 dst: /127.0.0.1:41033
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,449 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:34558 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34558 dst: /127.0.0.1:41033
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,450 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64527816{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:28,450 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:34548 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41033:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34548 dst: /127.0.0.1:41033
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:28,450 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1799acad{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:28,450 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:28,451 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28abb6ae{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:28,451 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d0d4ae6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:28,452 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:28,452 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:28,452 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid 614f64c7-b1fd-4be5-911a-2b5c55fb0913) service to localhost/127.0.0.1:46471
2024-11-22T17:36:28,452 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:28,452 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data1/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:28,453 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data2/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:28,453 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:28,461 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:28,464 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:28,465 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:28,465 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:28,465 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:36:28,466 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2500ff73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:28,466 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c02cba1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:28,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:28,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:28,573 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@745cef36{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-34167-hadoop-hdfs-3_4_1-tests_jar-_-any-4461974137035377490/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:28,574 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@23d5eda4{HTTP/1.1, (http/1.1)}{localhost:34167}
2024-11-22T17:36:28,574 INFO  [Time-limited test {}] server.Server(415): Started @174612ms
2024-11-22T17:36:28,575 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:28,988 WARN  [Thread-1351 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:28,992 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x331778f6fbf8c511 with lease ID 0x499f6377f563216: from storage DS-56c7e442-b398-4304-932f-9a60320019de node DatanodeRegistration(127.0.0.1:43945, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=39937, infoSecurePort=0, ipcPort=44929, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:28,992 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x331778f6fbf8c511 with lease ID 0x499f6377f563216: from storage DS-a5981da5-f013-4351-990a-2a585d6f9fc1 node DatanodeRegistration(127.0.0.1:43945, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=39937, infoSecurePort=0, ipcPort=44929, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:29,142 WARN  [Thread-1371 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:29,144 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cc566fed79037c8 with lease ID 0x499f6377f563217: from storage DS-46de46a3-2c67-46a6-a707-8ab286489e9f node DatanodeRegistration(127.0.0.1:35845, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=34527, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:36:29,144 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cc566fed79037c8 with lease ID 0x499f6377f563217: from storage DS-0ea4a2e9-d250-4493-b3af-88e30981425d node DatanodeRegistration(127.0.0.1:35845, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=34527, infoSecurePort=0, ipcPort=37691, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:29,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:29,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:29,597 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-22T17:36:29,599 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-22T17:36:29,600 ERROR [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:29,601 WARN  [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:29,601 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C44007%2C1732296974869:(num 1732296975533) roll requested
2024-11-22T17:36:29,601 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:29,608 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 newFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:29,609 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:29,609 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:29,609 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:29,609 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:29,609 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:29,609 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:29,609 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:29,610 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:29,610 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:29,610 WARN  [IPC Server handler 3 on default port 46471 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-11-22T17:36:29,610 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 after 0ms
2024-11-22T17:36:29,624 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34527:34527),(127.0.0.1/127.0.0.1:39937:39937)]
2024-11-22T17:36:29,624 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 is not closed yet, will try archiving it next time
2024-11-22T17:36:30,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:30,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:31,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:31,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:31,627 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-22T17:36:31,993 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T17:36:32,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:32,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:33,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:33,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:33,612 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 after 4001ms
2024-11-22T17:36:33,630 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:33,630 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35845,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK], DatanodeInfoWithStorage[127.0.0.1:43945,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35845,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]) is bad.
2024-11-22T17:36:33,631 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:35264 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:35845:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35264 dst: /127.0.0.1:35845
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:33,631 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:48538 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43945:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48538 dst: /127.0.0.1:43945
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:33,633 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@745cef36{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:33,633 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@23d5eda4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:33,633 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:33,633 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c02cba1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:33,634 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2500ff73{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:33,635 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:33,635 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:33,635 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:33,635 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid 614f64c7-b1fd-4be5-911a-2b5c55fb0913) service to localhost/127.0.0.1:46471
2024-11-22T17:36:33,636 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data1/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:33,636 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data2/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:33,636 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:33,644 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:33,648 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:33,648 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:33,648 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:33,649 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:33,649 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52b0383{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:33,649 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d6c3334{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:33,759 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ff00ad8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-33435-hadoop-hdfs-3_4_1-tests_jar-_-any-6941367584310564770/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:33,759 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3a287ee6{HTTP/1.1, (http/1.1)}{localhost:33435}
2024-11-22T17:36:33,759 INFO  [Time-limited test {}] server.Server(415): Started @179797ms
2024-11-22T17:36:33,760 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:33,783 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:33,784 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1863694155_22 at /127.0.0.1:48824 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43945:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48824 dst: /127.0.0.1:43945
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:33,789 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@68a7eef1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:33,790 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a86c024{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:33,790 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:33,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cda8c7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:33,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19409378{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:33,791 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:33,791 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:33,791 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid d9a2ad13-e794-44fe-bc3e-7b72309884b1) service to localhost/127.0.0.1:46471
2024-11-22T17:36:33,791 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:33,792 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data3/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:33,792 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data4/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:33,792 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:33,808 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:33,812 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:33,818 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:33,818 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:33,818 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:33,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1649c6f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:33,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@263c339e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:33,929 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@225d3925{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/java.io.tmpdir/jetty-localhost-46055-hadoop-hdfs-3_4_1-tests_jar-_-any-15896357140529098219/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:33,930 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22f6299d{HTTP/1.1, (http/1.1)}{localhost:46055}
2024-11-22T17:36:33,930 INFO  [Time-limited test {}] server.Server(415): Started @179967ms
2024-11-22T17:36:33,931 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:34,319 WARN  [Thread-1425 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:34,321 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677a48f0998ce3ee with lease ID 0x499f6377f563218: from storage DS-46de46a3-2c67-46a6-a707-8ab286489e9f node DatanodeRegistration(127.0.0.1:37991, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=36603, infoSecurePort=0, ipcPort=33029, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:34,322 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677a48f0998ce3ee with lease ID 0x499f6377f563218: from storage DS-0ea4a2e9-d250-4493-b3af-88e30981425d node DatanodeRegistration(127.0.0.1:37991, datanodeUuid=614f64c7-b1fd-4be5-911a-2b5c55fb0913, infoPort=36603, infoSecurePort=0, ipcPort=33029, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:34,476 WARN  [Thread-1445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:34,478 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4eb8b8e3efa320be with lease ID 0x499f6377f563219: from storage DS-56c7e442-b398-4304-932f-9a60320019de node DatanodeRegistration(127.0.0.1:39171, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=38029, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:34,478 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4eb8b8e3efa320be with lease ID 0x499f6377f563219: from storage DS-a5981da5-f013-4351-990a-2a585d6f9fc1 node DatanodeRegistration(127.0.0.1:39171, datanodeUuid=d9a2ad13-e794-44fe-bc3e-7b72309884b1, infoPort=38029, infoSecurePort=0, ipcPort=40313, storageInfo=lv=-57;cid=testClusterID;nsid=1577430636;c=1732296972497), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:34,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:34,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:34,950 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-22T17:36:34,952 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-22T17:36:34,954 ERROR [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43945,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:34,954 WARN  [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43945,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:34,954 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C44007%2C1732296974869:(num 1732296989601) roll requested
2024-11-22T17:36:34,955 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:34,961 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 newFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:34,962 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:34,962 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:34,962 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:34,962 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:34,962 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:34,962 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:34,963 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43945,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:34,963 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43945,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:34,963 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:34,963 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38029:38029),(127.0.0.1/127.0.0.1:36603:36603)]
2024-11-22T17:36:34,963 WARN  [IPC Server handler 2 on default port 46471 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-22T17:36:34,963 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 is not closed yet, will try archiving it next time
2024-11-22T17:36:34,964 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 after 1ms
2024-11-22T17:36:35,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:35,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:36,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:36,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:36,966 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:36,974 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955 newFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:36,975 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:36,975 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:36,975 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:36,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:36,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:36,976 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:36,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741838_1019 (size=1264)
2024-11-22T17:36:36,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741838_1019 (size=1264)
2024-11-22T17:36:36,981 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 is not closed yet, will try archiving it next time
2024-11-22T17:36:37,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36603:36603),(127.0.0.1/127.0.0.1:38029:38029)]
2024-11-22T17:36:37,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 is not closed yet, will try archiving it next time
2024-11-22T17:36:37,001 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:37,001 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:37,001 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533 after 0ms
2024-11-22T17:36:37,002 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:37,014 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732296976490/Put/vlen=218/seqid=0]
2024-11-22T17:36:37,014 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732296986215/Put/vlen=1045/seqid=0]
2024-11-22T17:36:37,015 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296975533
2024-11-22T17:36:37,015 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:37,015 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:37,015 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 after 0ms
2024-11-22T17:36:37,015 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:37,018 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732296989600/Put/vlen=1045/seqid=0]
2024-11-22T17:36:37,019 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732296991628/Put/vlen=1045/seqid=0]
2024-11-22T17:36:37,019 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601
2024-11-22T17:36:37,019 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:37,019 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:37,019 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955 after 0ms
2024-11-22T17:36:37,019 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296994955
2024-11-22T17:36:37,023 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732296994954/Put/vlen=1045/seqid=0]
2024-11-22T17:36:37,023 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:37,023 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:37,023 WARN  [IPC Server handler 4 on default port 46471 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-22T17:36:37,024 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 after 1ms
2024-11-22T17:36:37,484 WARN  [ResponseProcessor for block BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:37,484 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-443229131_22 at /127.0.0.1:37736 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37991:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37736 dst: /127.0.0.1:37991
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37991 remote=/127.0.0.1:37736]. Total timeout mills is 60000, 59490 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:37,484 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 block BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37991,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK], DatanodeInfoWithStorage[127.0.0.1:39171,DS-56c7e442-b398-4304-932f-9a60320019de,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37991,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]) is bad.
2024-11-22T17:36:37,484 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-443229131_22 at /127.0.0.1:40188 [Receiving block BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39171:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40188 dst: /127.0.0.1:39171
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:36:37,488 WARN  [DataStreamer for file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 block BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:37,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741839_1022 (size=85)
2024-11-22T17:36:37,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:37,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:38,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:38,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:38,965 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296989601 after 4002ms
2024-11-22T17:36:39,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:39,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:40,322 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T17:36:40,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:40,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:41,024 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 after 4001ms
2024-11-22T17:36:41,024 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:41,028 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:41,029 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing e65354aba8acff3f70c7d54b2ed437d4 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-22T17:36:41,029 ERROR [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,029 WARN  [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,030 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C44007%2C1732296974869:(num 1732296996965) roll requested
2024-11-22T17:36:41,030 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.1732297001030
2024-11-22T17:36:41,035 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 newFile=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732297001030
2024-11-22T17:36:41,035 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,036 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,036 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,036 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,036 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,036 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732297001030
2024-11-22T17:36:41,036 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,037 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-806922106-172.17.0.2-1732296972497:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,037 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:41,037 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 after 0ms
2024-11-22T17:36:41,049 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.1732296996965 to hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs/e61df7af61d2%2C44007%2C1732296974869.1732296996965
2024-11-22T17:36:41,049 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36603:36603),(127.0.0.1/127.0.0.1:38029:38029)]
2024-11-22T17:36:41,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/.tmp/info/13b4745079664177a31200857aae1726 is 1080, key is row1002/info:/1732296986215/Put/seqid=0
2024-11-22T17:36:41,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741841_1024 (size=9270)
2024-11-22T17:36:41,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741841_1024 (size=9270)
2024-11-22T17:36:41,087 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/.tmp/info/13b4745079664177a31200857aae1726
2024-11-22T17:36:41,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/.tmp/info/13b4745079664177a31200857aae1726 as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/info/13b4745079664177a31200857aae1726
2024-11-22T17:36:41,106 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/info/13b4745079664177a31200857aae1726, entries=4, sequenceid=8, filesize=9.1 K
2024-11-22T17:36:41,108 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for e65354aba8acff3f70c7d54b2ed437d4 in 80ms, sequenceid=8, compaction requested=false
2024-11-22T17:36:41,108 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e65354aba8acff3f70c7d54b2ed437d4:

2024-11-22T17:36:41,108 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-22T17:36:41,109 ERROR [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,109 WARN  [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153-prefix:e61df7af61d2,44007,1732296974869.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,109 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C44007%2C1732296974869.meta:.meta(num 1732296975921) roll requested
2024-11-22T17:36:41,109 INFO  [regionserver/e61df7af61d2:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C44007%2C1732296974869.meta.1732297001109.meta
2024-11-22T17:36:41,121 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,121 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,122 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732297001109.meta
2024-11-22T17:36:41,122 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,122 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:41,122 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta
2024-11-22T17:36:41,123 WARN  [IPC Server handler 2 on default port 46471 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-11-22T17:36:41,123 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta after 0ms
2024-11-22T17:36:41,128 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38029:38029),(127.0.0.1/127.0.0.1:36603:36603)]
2024-11-22T17:36:41,128 DEBUG [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta is not closed yet, will try archiving it next time
2024-11-22T17:36:41,151 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/info/aab5929a5c2e434794832177836f7cd4 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4./info:regioninfo/1732296976495/Put/seqid=0
2024-11-22T17:36:41,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741843_1027 (size=7125)
2024-11-22T17:36:41,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741843_1027 (size=7125)
2024-11-22T17:36:41,163 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/info/aab5929a5c2e434794832177836f7cd4
2024-11-22T17:36:41,191 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/ns/ad12c8c2ad3c4cdbb740b6004e921075 is 43, key is default/ns:d/1732296976028/Put/seqid=0
2024-11-22T17:36:41,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741844_1028 (size=5153)
2024-11-22T17:36:41,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741844_1028 (size=5153)
2024-11-22T17:36:41,199 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/ns/ad12c8c2ad3c4cdbb740b6004e921075
2024-11-22T17:36:41,226 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/table/bc48f53f7b35426b94a398b6709aa83d is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732296976507/Put/seqid=0
2024-11-22T17:36:41,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741845_1029 (size=5438)
2024-11-22T17:36:41,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741845_1029 (size=5438)
2024-11-22T17:36:41,234 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/table/bc48f53f7b35426b94a398b6709aa83d
2024-11-22T17:36:41,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/info/aab5929a5c2e434794832177836f7cd4 as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/info/aab5929a5c2e434794832177836f7cd4
2024-11-22T17:36:41,249 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/info/aab5929a5c2e434794832177836f7cd4, entries=10, sequenceid=11, filesize=7.0 K
2024-11-22T17:36:41,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/ns/ad12c8c2ad3c4cdbb740b6004e921075 as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/ns/ad12c8c2ad3c4cdbb740b6004e921075
2024-11-22T17:36:41,260 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/ns/ad12c8c2ad3c4cdbb740b6004e921075, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T17:36:41,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/.tmp/table/bc48f53f7b35426b94a398b6709aa83d as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/table/bc48f53f7b35426b94a398b6709aa83d
2024-11-22T17:36:41,270 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/table/bc48f53f7b35426b94a398b6709aa83d, entries=2, sequenceid=11, filesize=5.3 K
2024-11-22T17:36:41,271 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 163ms, sequenceid=11, compaction requested=false
2024-11-22T17:36:41,271 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T17:36:41,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:36:41,278 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:36:41,278 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:41,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:41,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:41,279 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:36:41,279 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:36:41,279 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=409849730, stopped=false
2024-11-22T17:36:41,279 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,45723,1732296974693
2024-11-22T17:36:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:41,407 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:36:41,407 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:36:41,407 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:41,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:41,408 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,44007,1732296974869' *****
2024-11-22T17:36:41,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:41,408 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:36:41,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:41,408 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:36:41,408 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:36:41,408 INFO  [RS:0;e61df7af61d2:44007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:36:41,408 INFO  [RS:0;e61df7af61d2:44007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(3091): Received CLOSE for e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,44007,1732296974869
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:44007.
2024-11-22T17:36:41,409 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e65354aba8acff3f70c7d54b2ed437d4, disabling compactions & flushes
2024-11-22T17:36:41,409 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:41,409 DEBUG [RS:0;e61df7af61d2:44007 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:36:41,409 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:41,409 DEBUG [RS:0;e61df7af61d2:44007 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:41,409 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4. after waiting 0 ms
2024-11-22T17:36:41,409 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:36:41,409 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:36:41,410 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:36:41,410 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T17:36:41,410 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:36:41,410 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:36:41,410 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:36:41,410 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:36:41,410 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:36:41,411 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1325): Online Regions={e65354aba8acff3f70c7d54b2ed437d4=TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4., 1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:36:41,411 DEBUG [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e65354aba8acff3f70c7d54b2ed437d4
2024-11-22T17:36:41,415 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/default/TestLogRolling-testLogRollOnPipelineRestart/e65354aba8acff3f70c7d54b2ed437d4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-22T17:36:41,415 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:41,416 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e65354aba8acff3f70c7d54b2ed437d4:
Waiting for close lock at 1732297001409Running coprocessor pre-close hooks at 1732297001409Disabling compacts and flushes for region at 1732297001409Disabling writes for close at 1732297001409Writing region close event to WAL at 1732297001410 (+1 ms)Running coprocessor post-close hooks at 1732297001415 (+5 ms)Closed at 1732297001415
2024-11-22T17:36:41,416 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732296976125.e65354aba8acff3f70c7d54b2ed437d4.
2024-11-22T17:36:41,417 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T17:36:41,417 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:36:41,417 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:41,417 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297001410Running coprocessor pre-close hooks at 1732297001410Disabling compacts and flushes for region at 1732297001410Disabling writes for close at 1732297001410Writing region close event to WAL at 1732297001412 (+2 ms)Running coprocessor post-close hooks at 1732297001417 (+5 ms)Closed at 1732297001417
2024-11-22T17:36:41,418 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:41,438 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-22T17:36:41,438 INFO  [regionserver/e61df7af61d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-22T17:36:41,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:41,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:41,611 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,44007,1732296974869; all regions closed.
2024-11-22T17:36:41,612 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,612 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,612 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,612 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,613 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:41,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741842_1025 (size=825)
2024-11-22T17:36:41,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741842_1025 (size=825)
2024-11-22T17:36:42,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:42,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:43,402 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:43,480 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T17:36:43,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:43,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:44,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:44,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:44,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:36:45,124 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta after 4001ms
2024-11-22T17:36:45,125 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/WALs/e61df7af61d2,44007,1732296974869/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta to hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs/e61df7af61d2%2C44007%2C1732296974869.meta.1732296975921.meta
2024-11-22T17:36:45,128 DEBUG [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs
2024-11-22T17:36:45,128 INFO  [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C44007%2C1732296974869.meta:.meta(num 1732297001109)
2024-11-22T17:36:45,129 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,129 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,129 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,129 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,130 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741840_1023 (size=1162)
2024-11-22T17:36:45,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741840_1023 (size=1162)
2024-11-22T17:36:45,137 DEBUG [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs
2024-11-22T17:36:45,137 INFO  [RS:0;e61df7af61d2:44007 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C44007%2C1732296974869:(num 1732297001030)
2024-11-22T17:36:45,137 DEBUG [RS:0;e61df7af61d2:44007 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:45,137 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:36:45,138 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:36:45,138 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:36:45,138 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:36:45,138 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:36:45,138 INFO  [RS:0;e61df7af61d2:44007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44007
2024-11-22T17:36:45,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:36:45,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,44007,1732296974869
2024-11-22T17:36:45,191 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:36:45,201 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,44007,1732296974869]
2024-11-22T17:36:45,211 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,44007,1732296974869 already deleted, retry=false
2024-11-22T17:36:45,212 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,44007,1732296974869 expired; onlineServers=0
2024-11-22T17:36:45,212 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,45723,1732296974693' *****
2024-11-22T17:36:45,212 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:36:45,212 INFO  [M:0;e61df7af61d2:45723 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:36:45,212 INFO  [M:0;e61df7af61d2:45723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:36:45,212 DEBUG [M:0;e61df7af61d2:45723 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:36:45,212 DEBUG [M:0;e61df7af61d2:45723 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:36:45,212 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:36:45,212 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296975273 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732296975273,5,FailOnTimeoutGroup]
2024-11-22T17:36:45,213 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296975273 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732296975273,5,FailOnTimeoutGroup]
2024-11-22T17:36:45,213 INFO  [M:0;e61df7af61d2:45723 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:36:45,213 INFO  [M:0;e61df7af61d2:45723 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:36:45,213 DEBUG [M:0;e61df7af61d2:45723 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:36:45,213 INFO  [M:0;e61df7af61d2:45723 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:36:45,213 INFO  [M:0;e61df7af61d2:45723 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:36:45,214 INFO  [M:0;e61df7af61d2:45723 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:36:45,214 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:36:45,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:36:45,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:45,222 DEBUG [M:0;e61df7af61d2:45723 {}] zookeeper.ZKUtil(347): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:36:45,222 WARN  [M:0;e61df7af61d2:45723 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:36:45,223 INFO  [M:0;e61df7af61d2:45723 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/.lastflushedseqids
2024-11-22T17:36:45,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741846_1030 (size=111)
2024-11-22T17:36:45,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741846_1030 (size=111)
2024-11-22T17:36:45,229 INFO  [M:0;e61df7af61d2:45723 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:36:45,229 INFO  [M:0;e61df7af61d2:45723 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:36:45,229 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:36:45,230 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:45,230 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:45,230 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:36:45,230 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:45,230 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-22T17:36:45,230 ERROR [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData-prefix:e61df7af61d2,45723,1732296974693 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:45,230 WARN  [FSHLog-0-hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData-prefix:e61df7af61d2,45723,1732296974693 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:45,230 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog e61df7af61d2%2C45723%2C1732296974693:(num 1732296975040) roll requested
2024-11-22T17:36:45,231 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C45723%2C1732296974693.1732297005231
2024-11-22T17:36:45,241 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,241 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,241 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,241 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,241 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,242 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732297005231
2024-11-22T17:36:45,242 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:45,242 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41033,DS-46de46a3-2c67-46a6-a707-8ab286489e9f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-22T17:36:45,242 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040
2024-11-22T17:36:45,242 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38029:38029),(127.0.0.1/127.0.0.1:36603:36603)]
2024-11-22T17:36:45,242 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 is not closed yet, will try archiving it next time
2024-11-22T17:36:45,243 WARN  [IPC Server handler 3 on default port 46471 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-22T17:36:45,243 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 after 1ms
2024-11-22T17:36:45,257 DEBUG [M:0;e61df7af61d2:45723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/473b0b91c3a448a382598e169e9b8616 is 82, key is hbase:meta,,1/info:regioninfo/1732296975952/Put/seqid=0
2024-11-22T17:36:45,266 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741848_1033 (size=5672)
2024-11-22T17:36:45,266 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741848_1033 (size=5672)
2024-11-22T17:36:45,267 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/473b0b91c3a448a382598e169e9b8616
2024-11-22T17:36:45,289 DEBUG [M:0;e61df7af61d2:45723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbfbc464d8424abf8ec856a9546d53ff is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732296976512/Put/seqid=0
2024-11-22T17:36:45,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741849_1034 (size=6118)
2024-11-22T17:36:45,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741849_1034 (size=6118)
2024-11-22T17:36:45,294 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbfbc464d8424abf8ec856a9546d53ff
2024-11-22T17:36:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:45,301 INFO  [RS:0;e61df7af61d2:44007 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:36:45,301 INFO  [RS:0;e61df7af61d2:44007 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,44007,1732296974869; zookeeper connection closed.
2024-11-22T17:36:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44007-0x101639e1cff0001, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:45,302 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52438865 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52438865
2024-11-22T17:36:45,302 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:36:45,318 DEBUG [M:0;e61df7af61d2:45723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d0f1bb128844335a0d08c0c05930611 is 69, key is e61df7af61d2,44007,1732296974869/rs:state/1732296975340/Put/seqid=0
2024-11-22T17:36:45,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741850_1035 (size=5156)
2024-11-22T17:36:45,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741850_1035 (size=5156)
2024-11-22T17:36:45,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:45,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:45,732 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d0f1bb128844335a0d08c0c05930611
2024-11-22T17:36:45,758 DEBUG [M:0;e61df7af61d2:45723 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3194bd65f28f4a0e946acbb9323dac8e is 52, key is load_balancer_on/state:d/1732296976119/Put/seqid=0
2024-11-22T17:36:45,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741851_1036 (size=5056)
2024-11-22T17:36:45,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741851_1036 (size=5056)
2024-11-22T17:36:45,764 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3194bd65f28f4a0e946acbb9323dac8e
2024-11-22T17:36:45,769 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/473b0b91c3a448a382598e169e9b8616 as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/473b0b91c3a448a382598e169e9b8616
2024-11-22T17:36:45,776 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/473b0b91c3a448a382598e169e9b8616, entries=8, sequenceid=56, filesize=5.5 K
2024-11-22T17:36:45,777 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbfbc464d8424abf8ec856a9546d53ff as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbfbc464d8424abf8ec856a9546d53ff
2024-11-22T17:36:45,783 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbfbc464d8424abf8ec856a9546d53ff, entries=6, sequenceid=56, filesize=6.0 K
2024-11-22T17:36:45,785 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0d0f1bb128844335a0d08c0c05930611 as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d0f1bb128844335a0d08c0c05930611
2024-11-22T17:36:45,791 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0d0f1bb128844335a0d08c0c05930611, entries=1, sequenceid=56, filesize=5.0 K
2024-11-22T17:36:45,792 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3194bd65f28f4a0e946acbb9323dac8e as hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3194bd65f28f4a0e946acbb9323dac8e
2024-11-22T17:36:45,798 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3194bd65f28f4a0e946acbb9323dac8e, entries=1, sequenceid=56, filesize=4.9 K
2024-11-22T17:36:45,799 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 569ms, sequenceid=56, compaction requested=false
2024-11-22T17:36:45,801 INFO  [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:45,801 DEBUG [M:0;e61df7af61d2:45723 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297005229Disabling compacts and flushes for region at 1732297005229Disabling writes for close at 1732297005230 (+1 ms)Obtaining lock to block concurrent updates at 1732297005230Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732297005230Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732297005230Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732297005243 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732297005243Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732297005257 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732297005257Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732297005273 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732297005288 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732297005288Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732297005300 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732297005317 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732297005317Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732297005740 (+423 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732297005757 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732297005757Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f37501b: reopening flushed file at 1732297005768 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e1388fa: reopening flushed file at 1732297005776 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5868ea02: reopening flushed file at 1732297005784 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52af3704: reopening flushed file at 1732297005791 (+7 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 569ms, sequenceid=56, compaction requested=false at 1732297005799 (+8 ms)Writing region close event to WAL at 1732297005801 (+2 ms)Closed at 1732297005801
2024-11-22T17:36:45,804 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,804 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,804 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,804 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,805 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:36:45,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37991 is added to blk_1073741847_1031 (size=757)
2024-11-22T17:36:45,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39171 is added to blk_1073741847_1031 (size=757)
2024-11-22T17:36:46,416 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,416 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,437 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,438 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,439 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,442 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,444 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,448 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,448 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,480 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-22T17:36:46,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:46,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:46,950 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:36:46,951 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,952 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,952 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,952 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,972 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,972 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,973 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,973 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,973 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,974 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,978 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,978 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,978 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:46,981 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:47,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:47,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:48,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:48,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:49,244 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 after 4002ms
2024-11-22T17:36:49,245 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/WALs/e61df7af61d2,45723,1732296974693/e61df7af61d2%2C45723%2C1732296974693.1732296975040 to hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/oldWALs/e61df7af61d2%2C45723%2C1732296974693.1732296975040
2024-11-22T17:36:49,249 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/MasterData/oldWALs/e61df7af61d2%2C45723%2C1732296974693.1732296975040 to hdfs://localhost:46471/user/jenkins/test-data/bc9039a0-f5b5-72bc-a1af-26545ce23153/oldWALs/e61df7af61d2%2C45723%2C1732296974693.1732296975040$masterlocalwal$
2024-11-22T17:36:49,249 INFO  [M:0;e61df7af61d2:45723 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:36:49,249 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:36:49,249 INFO  [M:0;e61df7af61d2:45723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45723
2024-11-22T17:36:49,249 INFO  [M:0;e61df7af61d2:45723 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:36:49,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:49,375 INFO  [M:0;e61df7af61d2:45723 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:36:49,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45723-0x101639e1cff0000, quorum=127.0.0.1:62961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:36:49,407 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@225d3925{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:49,408 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22f6299d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:49,408 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:49,408 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@263c339e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:49,408 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1649c6f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:49,410 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:49,410 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:49,410 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid d9a2ad13-e794-44fe-bc3e-7b72309884b1) service to localhost/127.0.0.1:46471
2024-11-22T17:36:49,410 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:49,411 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data3/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:49,411 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data4/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:49,412 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:49,414 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ff00ad8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:49,415 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3a287ee6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:49,415 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:49,415 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d6c3334{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:49,415 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52b0383{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:49,416 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:36:49,416 WARN  [BP-806922106-172.17.0.2-1732296972497 heartbeating to localhost/127.0.0.1:46471 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-806922106-172.17.0.2-1732296972497 (Datanode Uuid 614f64c7-b1fd-4be5-911a-2b5c55fb0913) service to localhost/127.0.0.1:46471
2024-11-22T17:36:49,416 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:36:49,416 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:36:49,416 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data1/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:49,416 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/cluster_ea248259-b4a7-10a9-5c5e-a1b4848bf41f/data/data2/current/BP-806922106-172.17.0.2-1732296972497 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:36:49,417 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:36:49,423 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@113238fc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:36:49,424 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c79a3ba{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:36:49,424 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:36:49,424 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7115368f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:36:49,424 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b850ed0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir/,STOPPED}
2024-11-22T17:36:49,430 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:36:49,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:36:49,458 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:46471
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46471
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46471 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46471
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46471 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46471
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46471 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46471
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=1763 (was 1935)
2024-11-22T17:36:49,468 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=1763
2024-11-22T17:36:49,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:36:49,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.log.dir so I do NOT create it in target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293
2024-11-22T17:36:49,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b93e329b-2974-3a3c-f91d-c04d7ba36566/hadoop.tmp.dir so I do NOT create it in target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead, deleteOnExit=true
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/test.cache.data in system properties and HBase conf
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:36:49,469 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:36:49,469 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:36:49,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:36:49,484 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:36:49,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:49,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:49,977 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:49,983 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:49,984 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:49,984 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:49,984 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:36:49,985 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:49,985 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38b36302{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:49,985 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e264d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:50,086 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58d2b6e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/java.io.tmpdir/jetty-localhost-34843-hadoop-hdfs-3_4_1-tests_jar-_-any-7109226837303243554/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:36:50,086 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14642123{HTTP/1.1, (http/1.1)}{localhost:34843}
2024-11-22T17:36:50,086 INFO  [Time-limited test {}] server.Server(415): Started @196124ms
2024-11-22T17:36:50,098 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:36:50,337 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:50,340 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:50,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:50,341 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:50,341 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:36:50,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f81b4d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:50,342 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@188d3049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:50,445 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cda66bc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/java.io.tmpdir/jetty-localhost-33477-hadoop-hdfs-3_4_1-tests_jar-_-any-7908455585534430175/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:50,446 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e84571d{HTTP/1.1, (http/1.1)}{localhost:33477}
2024-11-22T17:36:50,446 INFO  [Time-limited test {}] server.Server(415): Started @196483ms
2024-11-22T17:36:50,447 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:50,474 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:36:50,478 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:36:50,478 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:36:50,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:36:50,479 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:36:50,479 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@432bc536{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:36:50,479 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@718e5030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:36:50,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:50,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:50,586 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cb8e2e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/java.io.tmpdir/jetty-localhost-36715-hadoop-hdfs-3_4_1-tests_jar-_-any-10541483473574033595/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:36:50,586 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33803ea9{HTTP/1.1, (http/1.1)}{localhost:36715}
2024-11-22T17:36:50,586 INFO  [Time-limited test {}] server.Server(415): Started @196624ms
2024-11-22T17:36:50,588 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:36:51,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:36:51,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:36:51,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:36:51,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-22T17:36:51,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:51,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:51,776 WARN  [Thread-1665 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data1/current/BP-754602635-172.17.0.2-1732297009494/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:51,776 WARN  [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data2/current/BP-754602635-172.17.0.2-1732297009494/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:51,797 WARN  [Thread-1629 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:51,808 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e598666cae150b0 with lease ID 0x4199ab1d6c0af937: Processing first storage report for DS-58be6b54-a1b6-4e77-a17a-73b5620959a2 from datanode DatanodeRegistration(127.0.0.1:46721, datanodeUuid=ff990895-b0ce-44de-aea2-7a970cfb0d25, infoPort=34121, infoSecurePort=0, ipcPort=41305, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494)
2024-11-22T17:36:51,808 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e598666cae150b0 with lease ID 0x4199ab1d6c0af937: from storage DS-58be6b54-a1b6-4e77-a17a-73b5620959a2 node DatanodeRegistration(127.0.0.1:46721, datanodeUuid=ff990895-b0ce-44de-aea2-7a970cfb0d25, infoPort=34121, infoSecurePort=0, ipcPort=41305, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:51,808 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e598666cae150b0 with lease ID 0x4199ab1d6c0af937: Processing first storage report for DS-38cf3571-b9a3-4a62-a26e-a619ece3903b from datanode DatanodeRegistration(127.0.0.1:46721, datanodeUuid=ff990895-b0ce-44de-aea2-7a970cfb0d25, infoPort=34121, infoSecurePort=0, ipcPort=41305, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494)
2024-11-22T17:36:51,808 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e598666cae150b0 with lease ID 0x4199ab1d6c0af937: from storage DS-38cf3571-b9a3-4a62-a26e-a619ece3903b node DatanodeRegistration(127.0.0.1:46721, datanodeUuid=ff990895-b0ce-44de-aea2-7a970cfb0d25, infoPort=34121, infoSecurePort=0, ipcPort=41305, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:36:51,887 WARN  [Thread-1676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data3/current/BP-754602635-172.17.0.2-1732297009494/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:51,887 WARN  [Thread-1677 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data4/current/BP-754602635-172.17.0.2-1732297009494/current, will proceed with Du for space computation calculation, 
2024-11-22T17:36:51,916 WARN  [Thread-1652 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:36:51,923 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f8ffa7905448384 with lease ID 0x4199ab1d6c0af938: Processing first storage report for DS-251d79ea-45e4-43b9-8060-dc29e31ea01c from datanode DatanodeRegistration(127.0.0.1:37127, datanodeUuid=164851e0-1e59-4287-8cc0-4eb2aea30258, infoPort=39559, infoSecurePort=0, ipcPort=45217, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494)
2024-11-22T17:36:51,923 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f8ffa7905448384 with lease ID 0x4199ab1d6c0af938: from storage DS-251d79ea-45e4-43b9-8060-dc29e31ea01c node DatanodeRegistration(127.0.0.1:37127, datanodeUuid=164851e0-1e59-4287-8cc0-4eb2aea30258, infoPort=39559, infoSecurePort=0, ipcPort=45217, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:36:51,923 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f8ffa7905448384 with lease ID 0x4199ab1d6c0af938: Processing first storage report for DS-4fc5ac05-3e4e-4942-9f84-51cbe875d22b from datanode DatanodeRegistration(127.0.0.1:37127, datanodeUuid=164851e0-1e59-4287-8cc0-4eb2aea30258, infoPort=39559, infoSecurePort=0, ipcPort=45217, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494)
2024-11-22T17:36:51,923 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f8ffa7905448384 with lease ID 0x4199ab1d6c0af938: from storage DS-4fc5ac05-3e4e-4942-9f84-51cbe875d22b node DatanodeRegistration(127.0.0.1:37127, datanodeUuid=164851e0-1e59-4287-8cc0-4eb2aea30258, infoPort=39559, infoSecurePort=0, ipcPort=45217, storageInfo=lv=-57;cid=testClusterID;nsid=1053784881;c=1732297009494), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:36:52,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293
2024-11-22T17:36:52,027 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/zookeeper_0, clientPort=64472, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:36:52,028 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64472
2024-11-22T17:36:52,029 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,030 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:36:52,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:36:52,049 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94 with version=8
2024-11-22T17:36:52,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:36:52,050 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:36:52,051 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:36:52,052 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34443
2024-11-22T17:36:52,053 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34443 connecting to ZooKeeper ensemble=127.0.0.1:64472
2024-11-22T17:36:52,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344430x0, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:36:52,107 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34443-0x101639eaee50000 connected
2024-11-22T17:36:52,285 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,287 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,289 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:52,289 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94, hbase.cluster.distributed=false
2024-11-22T17:36:52,291 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:36:52,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34443
2024-11-22T17:36:52,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34443
2024-11-22T17:36:52,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34443
2024-11-22T17:36:52,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34443
2024-11-22T17:36:52,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34443
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:36:52,309 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:36:52,310 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37657
2024-11-22T17:36:52,312 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37657 connecting to ZooKeeper ensemble=127.0.0.1:64472
2024-11-22T17:36:52,313 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,315 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376570x0, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:36:52,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37657-0x101639eaee50001 connected
2024-11-22T17:36:52,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:36:52,328 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:36:52,328 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:36:52,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:36:52,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:36:52,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37657
2024-11-22T17:36:52,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37657
2024-11-22T17:36:52,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37657
2024-11-22T17:36:52,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37657
2024-11-22T17:36:52,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37657
2024-11-22T17:36:52,362 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:34443
2024-11-22T17:36:52,363 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:52,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:52,375 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:36:52,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,386 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:36:52,386 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,34443,1732297012050 from backup master directory
2024-11-22T17:36:52,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:52,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:36:52,396 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:36:52,396 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,400 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/hbase.id] with ID: 70b7e336-adc5-4677-80f7-11241c8b5049
2024-11-22T17:36:52,400 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/.tmp/hbase.id
2024-11-22T17:36:52,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:36:52,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:36:52,420 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/.tmp/hbase.id]:[hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/hbase.id]
2024-11-22T17:36:52,436 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:52,436 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:36:52,438 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-22T17:36:52,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:36:52,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:36:52,468 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:36:52,468 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:36:52,469 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:52,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:36:52,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:36:52,509 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store
2024-11-22T17:36:52,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:52,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:52,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:36:52,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:36:52,527 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:52,527 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:36:52,527 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297012527Disabling compacts and flushes for region at 1732297012527Disabling writes for close at 1732297012527Writing region close event to WAL at 1732297012527Closed at 1732297012527
2024-11-22T17:36:52,528 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/.initializing
2024-11-22T17:36:52,528 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/WALs/e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,531 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C34443%2C1732297012050, suffix=, logDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/WALs/e61df7af61d2,34443,1732297012050, archiveDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/oldWALs, maxLogs=10
2024-11-22T17:36:52,532 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C34443%2C1732297012050.1732297012532
2024-11-22T17:36:52,546 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/WALs/e61df7af61d2,34443,1732297012050/e61df7af61d2%2C34443%2C1732297012050.1732297012532
2024-11-22T17:36:52,560 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39559:39559),(127.0.0.1/127.0.0.1:34121:34121)]
2024-11-22T17:36:52,567 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:52,567 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:52,567 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,568 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,569 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:52,571 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:36:52,571 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,571 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:52,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:52,573 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:36:52,573 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:52,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:52,575 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:36:52,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:52,576 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:36:52,577 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:36:52,577 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,578 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:52,578 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,579 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,579 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,581 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,581 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,582 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:36:52,584 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:36:52,589 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:52,590 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=814903, jitterRate=0.036203980445861816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:36:52,591 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732297012568Initializing all the Stores at 1732297012569 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297012569Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297012569Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297012569Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297012569Cleaning up temporary data from old regions at 1732297012581 (+12 ms)Region opened successfully at 1732297012591 (+10 ms)
2024-11-22T17:36:52,592 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:36:52,596 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ec341b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:36:52,597 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:36:52,598 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:36:52,598 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:36:52,598 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:36:52,599 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:36:52,599 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:36:52,599 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:36:52,602 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:36:52,603 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:36:52,611 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:36:52,612 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:36:52,613 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:36:52,621 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:36:52,622 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:36:52,623 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:36:52,632 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:36:52,633 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:36:52,642 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:36:52,645 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:36:52,653 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:36:52,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:52,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:36:52,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,664 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,34443,1732297012050, sessionid=0x101639eaee50000, setting cluster-up flag (Was=false)
2024-11-22T17:36:52,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,716 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:36:52,717 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:52,769 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:36:52,770 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,34443,1732297012050
2024-11-22T17:36:52,771 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:36:52,772 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:52,773 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:36:52,773 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:36:52,773 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,34443,1732297012050
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:36:52,774 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732297042775
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:36:52,775 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:36:52,776 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,776 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:36:52,776 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:52,776 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:36:52,776 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:36:52,776 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:36:52,776 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:36:52,777 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:36:52,777 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297012777,5,FailOnTimeoutGroup]
2024-11-22T17:36:52,777 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297012777,5,FailOnTimeoutGroup]
2024-11-22T17:36:52,777 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,777 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:36:52,777 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,777 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,777 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,777 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:36:52,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:36:52,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:36:52,786 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:36:52,786 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94
2024-11-22T17:36:52,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:36:52,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:36:52,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:52,796 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:36:52,798 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:36:52,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,798 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:52,798 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:36:52,799 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:36:52,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,800 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:52,800 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:36:52,801 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:36:52,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,802 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:52,802 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:36:52,803 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:36:52,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:52,804 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:52,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:36:52,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740
2024-11-22T17:36:52,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740
2024-11-22T17:36:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:36:52,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:36:52,807 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:36:52,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:36:52,810 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:52,811 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872958, jitterRate=0.11002370715141296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:36:52,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732297012795Initializing all the Stores at 1732297012796 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297012796Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297012796Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297012796Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297012796Cleaning up temporary data from old regions at 1732297012806 (+10 ms)Region opened successfully at 1732297012812 (+6 ms)
2024-11-22T17:36:52,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:36:52,812 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:36:52,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:36:52,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:36:52,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:36:52,813 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:36:52,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297012812Disabling compacts and flushes for region at 1732297012812Disabling writes for close at 1732297012812Writing region close event to WAL at 1732297012813 (+1 ms)Closed at 1732297012813
2024-11-22T17:36:52,815 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:52,815 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:36:52,815 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:36:52,816 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:36:52,817 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:36:52,846 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(746): ClusterId : 70b7e336-adc5-4677-80f7-11241c8b5049
2024-11-22T17:36:52,846 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:36:52,860 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:36:52,860 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:36:52,870 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:36:52,870 DEBUG [RS:0;e61df7af61d2:37657 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284610eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:36:52,881 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:37657
2024-11-22T17:36:52,881 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:36:52,881 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:36:52,881 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:36:52,882 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,34443,1732297012050 with port=37657, startcode=1732297012309
2024-11-22T17:36:52,882 DEBUG [RS:0;e61df7af61d2:37657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:36:52,884 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:36:52,885 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34443 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,885 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34443 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,887 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94
2024-11-22T17:36:52,887 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44413
2024-11-22T17:36:52,887 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:36:52,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:36:52,895 DEBUG [RS:0;e61df7af61d2:37657 {}] zookeeper.ZKUtil(111): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,895 WARN  [RS:0;e61df7af61d2:37657 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:36:52,896 INFO  [RS:0;e61df7af61d2:37657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:52,896 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,37657,1732297012309]
2024-11-22T17:36:52,896 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,903 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:36:52,905 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:36:52,905 INFO  [RS:0;e61df7af61d2:37657 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:36:52,905 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,906 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:36:52,906 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:36:52,907 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:36:52,907 DEBUG [RS:0;e61df7af61d2:37657 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,908 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,37657,1732297012309-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:36:52,926 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:36:52,926 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,37657,1732297012309-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,926 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,926 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.Replication(171): e61df7af61d2,37657,1732297012309 started
2024-11-22T17:36:52,941 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:52,941 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,37657,1732297012309, RpcServer on e61df7af61d2/172.17.0.2:37657, sessionid=0x101639eaee50001
2024-11-22T17:36:52,941 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:36:52,941 DEBUG [RS:0;e61df7af61d2:37657 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,941 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,37657,1732297012309'
2024-11-22T17:36:52,941 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:36:52,942 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:36:52,943 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:36:52,943 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:36:52,943 DEBUG [RS:0;e61df7af61d2:37657 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,37657,1732297012309
2024-11-22T17:36:52,943 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,37657,1732297012309'
2024-11-22T17:36:52,943 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:36:52,944 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:36:52,944 DEBUG [RS:0;e61df7af61d2:37657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:36:52,944 INFO  [RS:0;e61df7af61d2:37657 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:36:52,944 INFO  [RS:0;e61df7af61d2:37657 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:36:52,968 WARN  [e61df7af61d2:34443 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:36:53,046 INFO  [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C37657%2C1732297012309, suffix=, logDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309, archiveDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs, maxLogs=32
2024-11-22T17:36:53,047 INFO  [RS:0;e61df7af61d2:37657 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C37657%2C1732297012309.1732297013046
2024-11-22T17:36:53,052 INFO  [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297013046
2024-11-22T17:36:53,053 DEBUG [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34121:34121),(127.0.0.1/127.0.0.1:39559:39559)]
2024-11-22T17:36:53,218 DEBUG [e61df7af61d2:34443 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:36:53,219 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:53,220 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,37657,1732297012309, state=OPENING
2024-11-22T17:36:53,232 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:36:53,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:53,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:36:53,243 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:36:53,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:53,243 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,37657,1732297012309}]
2024-11-22T17:36:53,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:53,397 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:36:53,399 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35371, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:36:53,403 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:36:53,403 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:36:53,405 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C37657%2C1732297012309.meta, suffix=.meta, logDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309, archiveDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs, maxLogs=32
2024-11-22T17:36:53,406 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C37657%2C1732297012309.meta.1732297013406.meta
2024-11-22T17:36:53,414 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.meta.1732297013406.meta
2024-11-22T17:36:53,416 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34121:34121),(127.0.0.1/127.0.0.1:39559:39559)]
2024-11-22T17:36:53,430 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:53,430 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:36:53,430 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:36:53,430 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:36:53,431 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:36:53,431 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:53,431 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:36:53,431 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:36:53,432 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:36:53,433 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:36:53,433 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,434 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:53,434 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:36:53,435 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:36:53,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:53,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:36:53,436 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:36:53,436 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,436 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:53,437 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:36:53,438 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:36:53,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,438 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:36:53,438 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:36:53,439 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740
2024-11-22T17:36:53,441 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740
2024-11-22T17:36:53,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:36:53,442 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:36:53,443 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:36:53,445 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:36:53,446 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836647, jitterRate=0.06385226547718048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:36:53,446 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:36:53,447 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732297013431Writing region info on filesystem at 1732297013431Initializing all the Stores at 1732297013432 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297013432Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297013432Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297013432Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297013432Cleaning up temporary data from old regions at 1732297013442 (+10 ms)Running coprocessor post-open hooks at 1732297013446 (+4 ms)Region opened successfully at 1732297013447 (+1 ms)
2024-11-22T17:36:53,448 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732297013397
2024-11-22T17:36:53,451 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:36:53,451 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:36:53,451 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:53,452 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,37657,1732297012309, state=OPEN
2024-11-22T17:36:53,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:36:53,490 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:53,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:36:53,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:53,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:36:53,493 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:36:53,494 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,37657,1732297012309 in 247 msec
2024-11-22T17:36:53,498 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:36:53,498 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 679 msec
2024-11-22T17:36:53,499 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:36:53,499 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:36:53,500 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:36:53,501 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,37657,1732297012309, seqNum=-1]
2024-11-22T17:36:53,501 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:36:53,503 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36757, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:36:53,511 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 737 msec
2024-11-22T17:36:53,511 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732297013511, completionTime=-1
2024-11-22T17:36:53,511 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:36:53,512 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732297073514
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297133514
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:34443, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,514 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,515 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,517 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.123sec
2024-11-22T17:36:53,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:36:53,519 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:36:53,522 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:36:53,522 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:36:53,522 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,34443,1732297012050-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:36:53,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:53,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ad6d55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:53,547 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,34443,-1 for getting cluster id
2024-11-22T17:36:53,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:36:53,549 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70b7e336-adc5-4677-80f7-11241c8b5049'
2024-11-22T17:36:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:36:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70b7e336-adc5-4677-80f7-11241c8b5049"

2024-11-22T17:36:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e03c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,34443,-1]
2024-11-22T17:36:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:36:53,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:36:53,552 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:36:53,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781dc17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:36:53,554 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:36:53,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,37657,1732297012309, seqNum=-1]
2024-11-22T17:36:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:36:53,557 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:36:53,559 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,34443,1732297012050
2024-11-22T17:36:53,560 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:36:53,563 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:36:53,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T17:36:53,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,34443,1732297012050
2024-11-22T17:36:53,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f022144
2024-11-22T17:36:53,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T17:36:53,566 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T17:36:53,567 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T17:36:53,567 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T17:36:53,567 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:36:53,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:36:53,570 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T17:36:53,570 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,571 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-22T17:36:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:36:53,572 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T17:36:53,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741835_1011 (size=405)
2024-11-22T17:36:53,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741835_1011 (size=405)
2024-11-22T17:36:53,588 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb7221be44cabb3375e9f5808856c88a, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94
2024-11-22T17:36:53,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741836_1012 (size=88)
2024-11-22T17:36:53,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741836_1012 (size=88)
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing eb7221be44cabb3375e9f5808856c88a, disabling compactions & flushes
2024-11-22T17:36:53,599 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a. after waiting 0 ms
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,599 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,599 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb7221be44cabb3375e9f5808856c88a:
Waiting for close lock at 1732297013599Disabling compacts and flushes for region at 1732297013599Disabling writes for close at 1732297013599Writing region close event to WAL at 1732297013599Closed at 1732297013599
2024-11-22T17:36:53,601 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T17:36:53,601 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732297013601"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732297013601"}]},"ts":"1732297013601"}
2024-11-22T17:36:53,604 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T17:36:53,605 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T17:36:53,605 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732297013605"}]},"ts":"1732297013605"}
2024-11-22T17:36:53,608 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-22T17:36:53,608 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=eb7221be44cabb3375e9f5808856c88a, ASSIGN}]
2024-11-22T17:36:53,610 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=eb7221be44cabb3375e9f5808856c88a, ASSIGN
2024-11-22T17:36:53,611 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=eb7221be44cabb3375e9f5808856c88a, ASSIGN; state=OFFLINE, location=e61df7af61d2,37657,1732297012309; forceNewPlan=false, retain=false
2024-11-22T17:36:53,761 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb7221be44cabb3375e9f5808856c88a, regionState=OPENING, regionLocation=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:53,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=eb7221be44cabb3375e9f5808856c88a, ASSIGN because future has completed
2024-11-22T17:36:53,766 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb7221be44cabb3375e9f5808856c88a, server=e61df7af61d2,37657,1732297012309}]
2024-11-22T17:36:53,923 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,924 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eb7221be44cabb3375e9f5808856c88a, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:36:53,924 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,924 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:36:53,924 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,924 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,925 INFO  [StoreOpener-eb7221be44cabb3375e9f5808856c88a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region eb7221be44cabb3375e9f5808856c88a 
2024-11-22T17:36:53,926 INFO  [StoreOpener-eb7221be44cabb3375e9f5808856c88a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb7221be44cabb3375e9f5808856c88a columnFamilyName info
2024-11-22T17:36:53,927 DEBUG [StoreOpener-eb7221be44cabb3375e9f5808856c88a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:36:53,927 INFO  [StoreOpener-eb7221be44cabb3375e9f5808856c88a-1 {}] regionserver.HStore(327): Store=eb7221be44cabb3375e9f5808856c88a/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:36:53,927 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,928 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,928 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,929 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,929 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,930 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,932 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:36:53,933 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eb7221be44cabb3375e9f5808856c88a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=867399, jitterRate=0.10295525193214417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:36:53,933 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:36:53,933 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eb7221be44cabb3375e9f5808856c88a:
Running coprocessor pre-open hook at 1732297013924Writing region info on filesystem at 1732297013924Initializing all the Stores at 1732297013925 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297013925Cleaning up temporary data from old regions at 1732297013929 (+4 ms)Running coprocessor post-open hooks at 1732297013933 (+4 ms)Region opened successfully at 1732297013933
2024-11-22T17:36:53,934 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a., pid=6, masterSystemTime=1732297013920
2024-11-22T17:36:53,937 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,937 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:36:53,938 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb7221be44cabb3375e9f5808856c88a, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,37657,1732297012309
2024-11-22T17:36:53,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb7221be44cabb3375e9f5808856c88a, server=e61df7af61d2,37657,1732297012309 because future has completed
2024-11-22T17:36:53,945 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T17:36:53,945 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eb7221be44cabb3375e9f5808856c88a, server=e61df7af61d2,37657,1732297012309 in 176 msec
2024-11-22T17:36:53,947 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T17:36:53,947 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=eb7221be44cabb3375e9f5808856c88a, ASSIGN in 337 msec
2024-11-22T17:36:53,948 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T17:36:53,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732297013949"}]},"ts":"1732297013949"}
2024-11-22T17:36:53,951 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-22T17:36:53,952 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T17:36:53,954 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 385 msec
2024-11-22T17:36:54,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:54,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:55,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:55,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:56,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:56,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:56,708 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:36:56,709 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,710 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,710 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,710 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,710 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,734 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,741 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:56,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:36:57,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:57,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:58,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:58,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:58,903 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T17:36:58,904 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-22T17:36:59,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:36:59,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:00,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:00,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:01,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-22T17:37:01,205 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-22T17:37:01,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:37:01,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-22T17:37:01,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-22T17:37:01,206 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-22T17:37:01,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:01,207 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T17:37:01,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:01,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:02,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:02,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:03,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:03,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:37:03,605 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T17:37:03,605 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-22T17:37:03,611 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:03,611 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:03,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a., hostname=e61df7af61d2,37657,1732297012309, seqNum=2]
2024-11-22T17:37:03,624 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:03,631 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T17:37:03,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-22T17:37:03,632 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T17:37:03,634 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T17:37:03,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-22T17:37:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:03,797 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing eb7221be44cabb3375e9f5808856c88a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:37:03,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/9ffdb6ca8ee64213a4352966de5fa3d9 is 1080, key is row0001/info:/1732297023616/Put/seqid=0
2024-11-22T17:37:03,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741837_1013 (size=6033)
2024-11-22T17:37:03,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741837_1013 (size=6033)
2024-11-22T17:37:03,822 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/9ffdb6ca8ee64213a4352966de5fa3d9
2024-11-22T17:37:03,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/9ffdb6ca8ee64213a4352966de5fa3d9 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9
2024-11-22T17:37:03,836 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9, entries=1, sequenceid=5, filesize=5.9 K
2024-11-22T17:37:03,838 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 40ms, sequenceid=5, compaction requested=false
2024-11-22T17:37:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for eb7221be44cabb3375e9f5808856c88a:

2024-11-22T17:37:03,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:03,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-22T17:37:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-22T17:37:03,847 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-22T17:37:03,847 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec
2024-11-22T17:37:03,849 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 222 msec
2024-11-22T17:37:04,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:04,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:05,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:05,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:06,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:06,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:07,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:07,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:08,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:08,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:09,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:09,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:10,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:10,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:11,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:11,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:12,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:12,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:13,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:13,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:13,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294 after 68053ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:37:13,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta after 68044ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-22T17:37:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-22T17:37:13,734 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T17:37:13,737 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-22T17:37:13,740 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T17:37:13,741 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T17:37:13,741 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T17:37:13,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-22T17:37:13,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:13,896 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing eb7221be44cabb3375e9f5808856c88a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:37:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a0f694adf5f14f2ea7ce0b4eb980d419 is 1080, key is row0002/info:/1732297033735/Put/seqid=0
2024-11-22T17:37:13,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741838_1014 (size=6033)
2024-11-22T17:37:13,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741838_1014 (size=6033)
2024-11-22T17:37:13,909 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a0f694adf5f14f2ea7ce0b4eb980d419
2024-11-22T17:37:13,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a0f694adf5f14f2ea7ce0b4eb980d419 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419
2024-11-22T17:37:13,923 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419, entries=1, sequenceid=9, filesize=5.9 K
2024-11-22T17:37:13,924 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 28ms, sequenceid=9, compaction requested=false
2024-11-22T17:37:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for eb7221be44cabb3375e9f5808856c88a:

2024-11-22T17:37:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-22T17:37:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-22T17:37:13,929 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-22T17:37:13,929 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 185 msec
2024-11-22T17:37:13,932 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 193 msec
2024-11-22T17:37:14,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:14,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:15,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:15,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:16,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:16,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:17,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:17,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:18,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:18,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:19,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:19,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:20,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:20,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:21,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:21,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:22,018 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:37:22,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:22,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:23,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:23,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-22T17:37:23,784 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T17:37:23,787 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C37657%2C1732297012309.1732297043787
2024-11-22T17:37:23,850 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:23,850 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:23,851 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:23,851 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:23,851 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:23,851 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297013046 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297043787
2024-11-22T17:37:23,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741833_1009 (size=5546)
2024-11-22T17:37:23,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741833_1009 (size=5546)
2024-11-22T17:37:23,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39559:39559),(127.0.0.1/127.0.0.1:34121:34121)]
2024-11-22T17:37:23,873 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-22T17:37:23,876 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T17:37:23,877 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T17:37:23,877 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T17:37:24,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-22T17:37:24,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:24,031 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing eb7221be44cabb3375e9f5808856c88a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:37:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/dd50327d850944db98eed74d6b4a1317 is 1080, key is row0003/info:/1732297043785/Put/seqid=0
2024-11-22T17:37:24,044 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741840_1016 (size=6033)
2024-11-22T17:37:24,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741840_1016 (size=6033)
2024-11-22T17:37:24,046 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/dd50327d850944db98eed74d6b4a1317
2024-11-22T17:37:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/dd50327d850944db98eed74d6b4a1317 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317
2024-11-22T17:37:24,057 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317, entries=1, sequenceid=13, filesize=5.9 K
2024-11-22T17:37:24,058 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 27ms, sequenceid=13, compaction requested=true
2024-11-22T17:37:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for eb7221be44cabb3375e9f5808856c88a:

2024-11-22T17:37:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-22T17:37:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-22T17:37:24,062 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-22T17:37:24,062 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec
2024-11-22T17:37:24,064 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-11-22T17:37:24,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:24,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:25,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:25,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:26,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:26,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:27,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:27,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:28,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:28,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:29,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:29,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:30,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:30,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:31,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:31,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:32,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:32,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:33,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:33,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-22T17:37:33,953 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T17:37:33,953 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:37:33,955 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:37:33,955 DEBUG [Time-limited test {}] regionserver.HStore(1541): eb7221be44cabb3375e9f5808856c88a/info is initiating minor compaction (all files)
2024-11-22T17:37:33,955 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:37:33,955 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:33,955 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of eb7221be44cabb3375e9f5808856c88a/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:33,955 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317] into tmpdir=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp, totalSize=17.7 K
2024-11-22T17:37:33,956 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9ffdb6ca8ee64213a4352966de5fa3d9, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732297023616
2024-11-22T17:37:33,956 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a0f694adf5f14f2ea7ce0b4eb980d419, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732297033735
2024-11-22T17:37:33,957 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dd50327d850944db98eed74d6b4a1317, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732297043785
2024-11-22T17:37:33,971 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): eb7221be44cabb3375e9f5808856c88a#info#compaction#46 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:37:33,972 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/697a3697668a4bdfa9677761fd80dec2 is 1080, key is row0001/info:/1732297023616/Put/seqid=0
2024-11-22T17:37:33,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741841_1017 (size=8296)
2024-11-22T17:37:33,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741841_1017 (size=8296)
2024-11-22T17:37:33,983 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/697a3697668a4bdfa9677761fd80dec2 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/697a3697668a4bdfa9677761fd80dec2
2024-11-22T17:37:33,990 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in eb7221be44cabb3375e9f5808856c88a/info of eb7221be44cabb3375e9f5808856c88a into 697a3697668a4bdfa9677761fd80dec2(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:37:33,990 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for eb7221be44cabb3375e9f5808856c88a:

2024-11-22T17:37:33,993 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C37657%2C1732297012309.1732297053993
2024-11-22T17:37:34,004 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:34,004 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:34,005 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:34,005 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:34,005 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:34,005 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297043787 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297053993
2024-11-22T17:37:34,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741839_1015 (size=2520)
2024-11-22T17:37:34,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741839_1015 (size=2520)
2024-11-22T17:37:34,008 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297013046 to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs/e61df7af61d2%2C37657%2C1732297012309.1732297013046
2024-11-22T17:37:34,008 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34121:34121),(127.0.0.1/127.0.0.1:39559:39559)]
2024-11-22T17:37:34,009 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-22T17:37:34,012 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-22T17:37:34,013 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-22T17:37:34,013 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-22T17:37:34,057 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T17:37:34,057 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T17:37:34,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-22T17:37:34,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:34,166 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing eb7221be44cabb3375e9f5808856c88a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:37:34,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/51bd51aa433d4987a3baf14265da5af4 is 1080, key is row0000/info:/1732297053992/Put/seqid=0
2024-11-22T17:37:34,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741843_1019 (size=6033)
2024-11-22T17:37:34,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741843_1019 (size=6033)
2024-11-22T17:37:34,177 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/51bd51aa433d4987a3baf14265da5af4
2024-11-22T17:37:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/51bd51aa433d4987a3baf14265da5af4 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/51bd51aa433d4987a3baf14265da5af4
2024-11-22T17:37:34,192 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/51bd51aa433d4987a3baf14265da5af4, entries=1, sequenceid=18, filesize=5.9 K
2024-11-22T17:37:34,193 INFO  [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 27ms, sequenceid=18, compaction requested=false
2024-11-22T17:37:34,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for eb7221be44cabb3375e9f5808856c88a:

2024-11-22T17:37:34,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:34,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-22T17:37:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-22T17:37:34,198 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-22T17:37:34,198 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec
2024-11-22T17:37:34,201 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec
2024-11-22T17:37:34,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:34,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:35,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:35,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:36,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:36,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:37,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:37,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:38,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:38,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:38,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region eb7221be44cabb3375e9f5808856c88a, had cached 0 bytes from a total of 14329
2024-11-22T17:37:39,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:39,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:40,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:40,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:41,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:41,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:42,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:42,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:43,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:43,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34443 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-22T17:37:44,064 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-22T17:37:44,068 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C37657%2C1732297012309.1732297064067
2024-11-22T17:37:44,078 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,078 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,078 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,078 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,078 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,079 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297053993 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297064067
2024-11-22T17:37:44,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39559:39559),(127.0.0.1/127.0.0.1:34121:34121)]
2024-11-22T17:37:44,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297053993 is not closed yet, will try archiving it next time
2024-11-22T17:37:44,080 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/WALs/e61df7af61d2,37657,1732297012309/e61df7af61d2%2C37657%2C1732297012309.1732297043787 to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs/e61df7af61d2%2C37657%2C1732297012309.1732297043787
2024-11-22T17:37:44,080 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:37:44,080 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:37:44,080 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:37:44,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:44,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741842_1018 (size=2026)
2024-11-22T17:37:44,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:44,081 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:37:44,081 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:37:44,081 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=843953927, stopped=false
2024-11-22T17:37:44,081 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,34443,1732297012050
2024-11-22T17:37:44,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741842_1018 (size=2026)
2024-11-22T17:37:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:37:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:44,102 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:37:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:37:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:44,102 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:37:44,102 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:37:44,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:44,102 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:37:44,103 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,37657,1732297012309' *****
2024-11-22T17:37:44,103 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:37:44,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:37:44,103 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:37:44,103 INFO  [RS:0;e61df7af61d2:37657 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:37:44,103 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:37:44,103 INFO  [RS:0;e61df7af61d2:37657 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:37:44,103 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(3091): Received CLOSE for eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,37657,1732297012309
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:37657.
2024-11-22T17:37:44,104 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb7221be44cabb3375e9f5808856c88a, disabling compactions & flushes
2024-11-22T17:37:44,104 DEBUG [RS:0;e61df7af61d2:37657 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:37:44,104 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:44,104 DEBUG [RS:0;e61df7af61d2:37657 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:44,104 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:44,104 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a. after waiting 0 ms
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:37:44,104 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:37:44,104 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:37:44,104 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eb7221be44cabb3375e9f5808856c88a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:37:44,105 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-22T17:37:44,105 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1325): Online Regions={eb7221be44cabb3375e9f5808856c88a=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a., 1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:37:44,105 DEBUG [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eb7221be44cabb3375e9f5808856c88a
2024-11-22T17:37:44,105 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:37:44,105 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:37:44,105 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:37:44,105 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:37:44,105 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:37:44,105 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-22T17:37:44,110 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a5c3c8dce49446289cee6979880a6dbc is 1080, key is row0001/info:/1732297064066/Put/seqid=0
2024-11-22T17:37:44,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741845_1021 (size=6033)
2024-11-22T17:37:44,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741845_1021 (size=6033)
2024-11-22T17:37:44,118 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a5c3c8dce49446289cee6979880a6dbc
2024-11-22T17:37:44,125 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/.tmp/info/a5c3c8dce49446289cee6979880a6dbc as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a5c3c8dce49446289cee6979880a6dbc
2024-11-22T17:37:44,125 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/info/37b790a3752f4f14b813ae35f9d46c9a is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a./info:regioninfo/1732297013938/Put/seqid=0
2024-11-22T17:37:44,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741846_1022 (size=7308)
2024-11-22T17:37:44,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741846_1022 (size=7308)
2024-11-22T17:37:44,138 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/info/37b790a3752f4f14b813ae35f9d46c9a
2024-11-22T17:37:44,139 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a5c3c8dce49446289cee6979880a6dbc, entries=1, sequenceid=22, filesize=5.9 K
2024-11-22T17:37:44,140 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 36ms, sequenceid=22, compaction requested=true
2024-11-22T17:37:44,141 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317] to archive
2024-11-22T17:37:44,142 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:37:44,144 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9 to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/9ffdb6ca8ee64213a4352966de5fa3d9
2024-11-22T17:37:44,145 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419 to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/a0f694adf5f14f2ea7ce0b4eb980d419
2024-11-22T17:37:44,147 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317 to hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/info/dd50327d850944db98eed74d6b4a1317
2024-11-22T17:37:44,147 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e61df7af61d2:34443 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T17:37:44,147 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [9ffdb6ca8ee64213a4352966de5fa3d9=6033, a0f694adf5f14f2ea7ce0b4eb980d419=6033, dd50327d850944db98eed74d6b4a1317=6033]
2024-11-22T17:37:44,151 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/eb7221be44cabb3375e9f5808856c88a/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-22T17:37:44,152 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:44,152 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb7221be44cabb3375e9f5808856c88a:
Waiting for close lock at 1732297064104Running coprocessor pre-close hooks at 1732297064104Disabling compacts and flushes for region at 1732297064104Disabling writes for close at 1732297064104Obtaining lock to block concurrent updates at 1732297064104Preparing flush snapshotting stores in eb7221be44cabb3375e9f5808856c88a at 1732297064104Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732297064105 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a. at 1732297064106 (+1 ms)Flushing eb7221be44cabb3375e9f5808856c88a/info: creating writer at 1732297064106Flushing eb7221be44cabb3375e9f5808856c88a/info: appending metadata at 1732297064109 (+3 ms)Flushing eb7221be44cabb3375e9f5808856c88a/info: closing flushed file at 1732297064109Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d105ac0: reopening flushed file at 1732297064124 (+15 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for eb7221be44cabb3375e9f5808856c88a in 36ms, sequenceid=22, compaction requested=true at 1732297064140 (+16 ms)Writing region close event to WAL at 1732297064148 (+8 ms)Running coprocessor post-close hooks at 1732297064152 (+4 ms)Closed at 1732297064152
2024-11-22T17:37:44,152 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732297013567.eb7221be44cabb3375e9f5808856c88a.
2024-11-22T17:37:44,165 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/ns/ced6a88ca3dc4a808864b1da922587eb is 43, key is default/ns:d/1732297013504/Put/seqid=0
2024-11-22T17:37:44,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741847_1023 (size=5153)
2024-11-22T17:37:44,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741847_1023 (size=5153)
2024-11-22T17:37:44,170 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/ns/ced6a88ca3dc4a808864b1da922587eb
2024-11-22T17:37:44,191 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/table/9847ec024230470fa7cae4e8b422afb4 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732297013949/Put/seqid=0
2024-11-22T17:37:44,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741848_1024 (size=5508)
2024-11-22T17:37:44,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741848_1024 (size=5508)
2024-11-22T17:37:44,198 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/table/9847ec024230470fa7cae4e8b422afb4
2024-11-22T17:37:44,204 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/info/37b790a3752f4f14b813ae35f9d46c9a as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/info/37b790a3752f4f14b813ae35f9d46c9a
2024-11-22T17:37:44,210 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/info/37b790a3752f4f14b813ae35f9d46c9a, entries=10, sequenceid=11, filesize=7.1 K
2024-11-22T17:37:44,211 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/ns/ced6a88ca3dc4a808864b1da922587eb as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/ns/ced6a88ca3dc4a808864b1da922587eb
2024-11-22T17:37:44,217 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/ns/ced6a88ca3dc4a808864b1da922587eb, entries=2, sequenceid=11, filesize=5.0 K
2024-11-22T17:37:44,218 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/.tmp/table/9847ec024230470fa7cae4e8b422afb4 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/table/9847ec024230470fa7cae4e8b422afb4
2024-11-22T17:37:44,226 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/table/9847ec024230470fa7cae4e8b422afb4, entries=2, sequenceid=11, filesize=5.4 K
2024-11-22T17:37:44,227 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=11, compaction requested=false
2024-11-22T17:37:44,232 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-22T17:37:44,233 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:37:44,233 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:37:44,233 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297064105Running coprocessor pre-close hooks at 1732297064105Disabling compacts and flushes for region at 1732297064105Disabling writes for close at 1732297064105Obtaining lock to block concurrent updates at 1732297064105Preparing flush snapshotting stores in 1588230740 at 1732297064105Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732297064105Flushing stores of hbase:meta,,1.1588230740 at 1732297064106 (+1 ms)Flushing 1588230740/info: creating writer at 1732297064106Flushing 1588230740/info: appending metadata at 1732297064125 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732297064125Flushing 1588230740/ns: creating writer at 1732297064147 (+22 ms)Flushing 1588230740/ns: appending metadata at 1732297064165 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732297064165Flushing 1588230740/table: creating writer at 1732297064175 (+10 ms)Flushing 1588230740/table: appending metadata at 1732297064190 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732297064190Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ae1883b: reopening flushed file at 1732297064203 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c348c6: reopening flushed file at 1732297064210 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bbfb0cb: reopening flushed file at 1732297064218 (+8 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=11, compaction requested=false at 1732297064227 (+9 ms)Writing region close event to WAL at 1732297064228 (+1 ms)Running coprocessor post-close hooks at 1732297064233 (+5 ms)Closed at 1732297064233
2024-11-22T17:37:44,233 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:37:44,305 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,37657,1732297012309; all regions closed.
2024-11-22T17:37:44,305 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,306 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,306 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,306 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,306 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741834_1010 (size=3306)
2024-11-22T17:37:44,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741834_1010 (size=3306)
2024-11-22T17:37:44,310 DEBUG [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs
2024-11-22T17:37:44,310 INFO  [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C37657%2C1732297012309.meta:.meta(num 1732297013406)
2024-11-22T17:37:44,310 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,310 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,311 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,311 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,311 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741844_1020 (size=1252)
2024-11-22T17:37:44,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741844_1020 (size=1252)
2024-11-22T17:37:44,316 DEBUG [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/oldWALs
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C37657%2C1732297012309:(num 1732297064067)
2024-11-22T17:37:44,316 DEBUG [RS:0;e61df7af61d2:37657 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:37:44,316 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:37:44,316 INFO  [RS:0;e61df7af61d2:37657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37657
2024-11-22T17:37:44,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,37657,1732297012309
2024-11-22T17:37:44,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:37:44,323 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:37:44,323 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,37657,1732297012309]
2024-11-22T17:37:44,417 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,37657,1732297012309 already deleted, retry=false
2024-11-22T17:37:44,417 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,37657,1732297012309 expired; onlineServers=0
2024-11-22T17:37:44,417 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,34443,1732297012050' *****
2024-11-22T17:37:44,417 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:37:44,417 INFO  [M:0;e61df7af61d2:34443 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:37:44,418 INFO  [M:0;e61df7af61d2:34443 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:37:44,418 DEBUG [M:0;e61df7af61d2:34443 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:37:44,418 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:37:44,418 DEBUG [M:0;e61df7af61d2:34443 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:37:44,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297012777 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297012777,5,FailOnTimeoutGroup]
2024-11-22T17:37:44,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297012777 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297012777,5,FailOnTimeoutGroup]
2024-11-22T17:37:44,418 INFO  [M:0;e61df7af61d2:34443 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:37:44,418 INFO  [M:0;e61df7af61d2:34443 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:37:44,418 DEBUG [M:0;e61df7af61d2:34443 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:37:44,418 INFO  [M:0;e61df7af61d2:34443 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:37:44,418 INFO  [M:0;e61df7af61d2:34443 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:37:44,419 INFO  [M:0;e61df7af61d2:34443 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:37:44,419 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:37:44,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:37:44,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:44,428 DEBUG [M:0;e61df7af61d2:34443 {}] zookeeper.ZKUtil(347): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:37:44,428 WARN  [M:0;e61df7af61d2:34443 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:37:44,429 INFO  [M:0;e61df7af61d2:34443 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/.lastflushedseqids
2024-11-22T17:37:44,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:37:44,433 INFO  [RS:0;e61df7af61d2:37657 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:37:44,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37657-0x101639eaee50001, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:37:44,433 INFO  [RS:0;e61df7af61d2:37657 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,37657,1732297012309; zookeeper connection closed.
2024-11-22T17:37:44,434 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3425f758 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3425f758
2024-11-22T17:37:44,434 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:37:44,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741849_1025 (size=130)
2024-11-22T17:37:44,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741849_1025 (size=130)
2024-11-22T17:37:44,439 INFO  [M:0;e61df7af61d2:34443 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:37:44,439 INFO  [M:0;e61df7af61d2:34443 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:37:44,439 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:37:44,439 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:44,439 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:44,439 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:37:44,440 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:44,440 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.59 KB heapSize=55 KB
2024-11-22T17:37:44,457 DEBUG [M:0;e61df7af61d2:34443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/567caaf4fd784a8b8f92373b6c72579c is 82, key is hbase:meta,,1/info:regioninfo/1732297013451/Put/seqid=0
2024-11-22T17:37:44,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741850_1026 (size=5672)
2024-11-22T17:37:44,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741850_1026 (size=5672)
2024-11-22T17:37:44,462 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/567caaf4fd784a8b8f92373b6c72579c
2024-11-22T17:37:44,482 DEBUG [M:0;e61df7af61d2:34443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/565aeb4bec5645c49e9b03dc316af135 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732297013953/Put/seqid=0
2024-11-22T17:37:44,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741851_1027 (size=7823)
2024-11-22T17:37:44,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741851_1027 (size=7823)
2024-11-22T17:37:44,487 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.99 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/565aeb4bec5645c49e9b03dc316af135
2024-11-22T17:37:44,493 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 565aeb4bec5645c49e9b03dc316af135
2024-11-22T17:37:44,514 DEBUG [M:0;e61df7af61d2:34443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e504c136ac4463db062bff487003232 is 69, key is e61df7af61d2,37657,1732297012309/rs:state/1732297012885/Put/seqid=0
2024-11-22T17:37:44,518 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741852_1028 (size=5156)
2024-11-22T17:37:44,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741852_1028 (size=5156)
2024-11-22T17:37:44,519 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e504c136ac4463db062bff487003232
2024-11-22T17:37:44,540 DEBUG [M:0;e61df7af61d2:34443 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d54f41db08e64913a5a48910df0c697f is 52, key is load_balancer_on/state:d/1732297013562/Put/seqid=0
2024-11-22T17:37:44,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741853_1029 (size=5056)
2024-11-22T17:37:44,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741853_1029 (size=5056)
2024-11-22T17:37:44,547 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d54f41db08e64913a5a48910df0c697f
2024-11-22T17:37:44,553 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/567caaf4fd784a8b8f92373b6c72579c as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/567caaf4fd784a8b8f92373b6c72579c
2024-11-22T17:37:44,559 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/567caaf4fd784a8b8f92373b6c72579c, entries=8, sequenceid=121, filesize=5.5 K
2024-11-22T17:37:44,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:44,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:44,560 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/565aeb4bec5645c49e9b03dc316af135 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/565aeb4bec5645c49e9b03dc316af135
2024-11-22T17:37:44,566 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 565aeb4bec5645c49e9b03dc316af135
2024-11-22T17:37:44,566 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/565aeb4bec5645c49e9b03dc316af135, entries=14, sequenceid=121, filesize=7.6 K
2024-11-22T17:37:44,567 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4e504c136ac4463db062bff487003232 as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4e504c136ac4463db062bff487003232
2024-11-22T17:37:44,573 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4e504c136ac4463db062bff487003232, entries=1, sequenceid=121, filesize=5.0 K
2024-11-22T17:37:44,574 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d54f41db08e64913a5a48910df0c697f as hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d54f41db08e64913a5a48910df0c697f
2024-11-22T17:37:44,580 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44413/user/jenkins/test-data/f34eb69b-527d-082a-15eb-1dcef9ff4a94/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d54f41db08e64913a5a48910df0c697f, entries=1, sequenceid=121, filesize=4.9 K
2024-11-22T17:37:44,581 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.59 KB/44638, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=121, compaction requested=false
2024-11-22T17:37:44,582 INFO  [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:44,582 DEBUG [M:0;e61df7af61d2:34443 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297064439Disabling compacts and flushes for region at 1732297064439Disabling writes for close at 1732297064439Obtaining lock to block concurrent updates at 1732297064440 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732297064440Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44638, getHeapSize=56256, getOffHeapSize=0, getCellsCount=140 at 1732297064440Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732297064441 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732297064441Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732297064456 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732297064456Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732297064466 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732297064481 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732297064481Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732297064493 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732297064513 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732297064513Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732297064524 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732297064539 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732297064539Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4215f4c9: reopening flushed file at 1732297064552 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49f4524e: reopening flushed file at 1732297064559 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@243b5de9: reopening flushed file at 1732297064566 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@345a9af1: reopening flushed file at 1732297064573 (+7 ms)Finished flush of dataSize ~43.59 KB/44638, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=121, compaction requested=false at 1732297064581 (+8 ms)Writing region close event to WAL at 1732297064582 (+1 ms)Closed at 1732297064582
2024-11-22T17:37:44,583 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,583 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,583 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,583 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,583 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:37:44,585 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46721 is added to blk_1073741830_1006 (size=53035)
2024-11-22T17:37:44,585 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37127 is added to blk_1073741830_1006 (size=53035)
2024-11-22T17:37:44,586 INFO  [M:0;e61df7af61d2:34443 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:37:44,586 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:37:44,586 INFO  [M:0;e61df7af61d2:34443 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34443
2024-11-22T17:37:44,586 INFO  [M:0;e61df7af61d2:34443 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:37:44,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:37:44,696 INFO  [M:0;e61df7af61d2:34443 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:37:44,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34443-0x101639eaee50000, quorum=127.0.0.1:64472, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:37:44,700 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cb8e2e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:37:44,700 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33803ea9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:37:44,700 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:37:44,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@718e5030{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:37:44,701 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@432bc536{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,STOPPED}
2024-11-22T17:37:44,703 WARN  [BP-754602635-172.17.0.2-1732297009494 heartbeating to localhost/127.0.0.1:44413 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:37:44,703 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:37:44,703 WARN  [BP-754602635-172.17.0.2-1732297009494 heartbeating to localhost/127.0.0.1:44413 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-754602635-172.17.0.2-1732297009494 (Datanode Uuid 164851e0-1e59-4287-8cc0-4eb2aea30258) service to localhost/127.0.0.1:44413
2024-11-22T17:37:44,703 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:37:44,704 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data3/current/BP-754602635-172.17.0.2-1732297009494 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:37:44,704 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data4/current/BP-754602635-172.17.0.2-1732297009494 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:37:44,704 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:37:44,707 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cda66bc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:37:44,707 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e84571d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:37:44,707 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:37:44,707 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@188d3049{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:37:44,708 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f81b4d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,STOPPED}
2024-11-22T17:37:44,709 WARN  [BP-754602635-172.17.0.2-1732297009494 heartbeating to localhost/127.0.0.1:44413 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:37:44,709 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:37:44,709 WARN  [BP-754602635-172.17.0.2-1732297009494 heartbeating to localhost/127.0.0.1:44413 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-754602635-172.17.0.2-1732297009494 (Datanode Uuid ff990895-b0ce-44de-aea2-7a970cfb0d25) service to localhost/127.0.0.1:44413
2024-11-22T17:37:44,709 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:37:44,710 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data1/current/BP-754602635-172.17.0.2-1732297009494 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:37:44,710 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/cluster_dde321f9-dd14-5853-14a5-e2a31310bead/data/data2/current/BP-754602635-172.17.0.2-1732297009494 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:37:44,710 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:37:44,716 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58d2b6e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:37:44,716 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14642123{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:37:44,717 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:37:44,717 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e264d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:37:44,717 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38b36302{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir/,STOPPED}
2024-11-22T17:37:44,723 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:37:44,748 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:37:44,757 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 179)
Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44413
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44413 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44413 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44413 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44413
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44413
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:44413
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44413
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=188 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=1641 (was 1763)
2024-11-22T17:37:44,765 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=188, ProcessCount=11, AvailableMemoryMB=1641
2024-11-22T17:37:44,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:37:44,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.log.dir so I do NOT create it in target/test-data/2e35c859-f2be-a247-79fa-328148e093d1
2024-11-22T17:37:44,765 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7aa1c0f8-705c-8faf-a787-88ea03bbb293/hadoop.tmp.dir so I do NOT create it in target/test-data/2e35c859-f2be-a247-79fa-328148e093d1
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2, deleteOnExit=true
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/test.cache.data in system properties and HBase conf
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:37:44,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:37:44,766 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:37:44,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:37:44,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:37:44,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:37:44,768 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:37:44,783 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:37:44,911 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:37:45,124 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:37:45,127 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:37:45,128 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:37:45,128 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:37:45,128 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-22T17:37:45,129 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:37:45,129 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6150e164{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:37:45,130 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ec6f4ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:37:45,233 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5350cad{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/java.io.tmpdir/jetty-localhost-42621-hadoop-hdfs-3_4_1-tests_jar-_-any-13708983944533454550/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:37:45,234 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@278f505{HTTP/1.1, (http/1.1)}{localhost:42621}
2024-11-22T17:37:45,234 INFO  [Time-limited test {}] server.Server(415): Started @251271ms
2024-11-22T17:37:45,246 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:37:45,500 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:37:45,504 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:37:45,504 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:37:45,504 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:37:45,505 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:37:45,505 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef16bcf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:37:45,505 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a5fe763{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:37:45,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:45,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:45,609 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@278c53c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/java.io.tmpdir/jetty-localhost-46759-hadoop-hdfs-3_4_1-tests_jar-_-any-14519111159476446984/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:37:45,609 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a127189{HTTP/1.1, (http/1.1)}{localhost:46759}
2024-11-22T17:37:45,609 INFO  [Time-limited test {}] server.Server(415): Started @251647ms
2024-11-22T17:37:45,611 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:37:45,641 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:37:45,644 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:37:45,645 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:37:45,645 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:37:45,645 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:37:45,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51d47a1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:37:45,646 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dbdcf8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:37:45,752 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4cfa1960{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/java.io.tmpdir/jetty-localhost-32777-hadoop-hdfs-3_4_1-tests_jar-_-any-13173110508174404708/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:37:45,752 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7289fa1e{HTTP/1.1, (http/1.1)}{localhost:32777}
2024-11-22T17:37:45,752 INFO  [Time-limited test {}] server.Server(415): Started @251790ms
2024-11-22T17:37:45,753 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:37:46,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:46,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:46,783 WARN  [Thread-1982 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data1/current/BP-413928561-172.17.0.2-1732297064796/current, will proceed with Du for space computation calculation, 
2024-11-22T17:37:46,783 WARN  [Thread-1983 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data2/current/BP-413928561-172.17.0.2-1732297064796/current, will proceed with Du for space computation calculation, 
2024-11-22T17:37:46,804 WARN  [Thread-1946 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:37:46,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb5a3def3d8883f69 with lease ID 0x81c93e8e8ab5ad3b: Processing first storage report for DS-71165548-6763-4bd9-8937-542888f22c7c from datanode DatanodeRegistration(127.0.0.1:44599, datanodeUuid=837ef63f-5c78-4b25-be8f-5aa961fe1140, infoPort=40237, infoSecurePort=0, ipcPort=46475, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796)
2024-11-22T17:37:46,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5a3def3d8883f69 with lease ID 0x81c93e8e8ab5ad3b: from storage DS-71165548-6763-4bd9-8937-542888f22c7c node DatanodeRegistration(127.0.0.1:44599, datanodeUuid=837ef63f-5c78-4b25-be8f-5aa961fe1140, infoPort=40237, infoSecurePort=0, ipcPort=46475, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-22T17:37:46,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb5a3def3d8883f69 with lease ID 0x81c93e8e8ab5ad3b: Processing first storage report for DS-714d4338-f11e-45c3-b17a-85ea745eb76d from datanode DatanodeRegistration(127.0.0.1:44599, datanodeUuid=837ef63f-5c78-4b25-be8f-5aa961fe1140, infoPort=40237, infoSecurePort=0, ipcPort=46475, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796)
2024-11-22T17:37:46,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb5a3def3d8883f69 with lease ID 0x81c93e8e8ab5ad3b: from storage DS-714d4338-f11e-45c3-b17a-85ea745eb76d node DatanodeRegistration(127.0.0.1:44599, datanodeUuid=837ef63f-5c78-4b25-be8f-5aa961fe1140, infoPort=40237, infoSecurePort=0, ipcPort=46475, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:37:46,900 WARN  [Thread-1993 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data3/current/BP-413928561-172.17.0.2-1732297064796/current, will proceed with Du for space computation calculation, 
2024-11-22T17:37:46,900 WARN  [Thread-1994 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data4/current/BP-413928561-172.17.0.2-1732297064796/current, will proceed with Du for space computation calculation, 
2024-11-22T17:37:46,926 WARN  [Thread-1969 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:37:46,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc25fe2be409acfa7 with lease ID 0x81c93e8e8ab5ad3c: Processing first storage report for DS-0a5ac07d-9b74-463e-b77e-250e3df37433 from datanode DatanodeRegistration(127.0.0.1:36371, datanodeUuid=d9274d56-39bc-4353-b62d-b734295cb6a3, infoPort=33497, infoSecurePort=0, ipcPort=35551, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796)
2024-11-22T17:37:46,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc25fe2be409acfa7 with lease ID 0x81c93e8e8ab5ad3c: from storage DS-0a5ac07d-9b74-463e-b77e-250e3df37433 node DatanodeRegistration(127.0.0.1:36371, datanodeUuid=d9274d56-39bc-4353-b62d-b734295cb6a3, infoPort=33497, infoSecurePort=0, ipcPort=35551, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:37:46,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc25fe2be409acfa7 with lease ID 0x81c93e8e8ab5ad3c: Processing first storage report for DS-80152ebc-1d6e-46cd-8025-e5c5602f0e4b from datanode DatanodeRegistration(127.0.0.1:36371, datanodeUuid=d9274d56-39bc-4353-b62d-b734295cb6a3, infoPort=33497, infoSecurePort=0, ipcPort=35551, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796)
2024-11-22T17:37:46,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc25fe2be409acfa7 with lease ID 0x81c93e8e8ab5ad3c: from storage DS-80152ebc-1d6e-46cd-8025-e5c5602f0e4b node DatanodeRegistration(127.0.0.1:36371, datanodeUuid=d9274d56-39bc-4353-b62d-b734295cb6a3, infoPort=33497, infoSecurePort=0, ipcPort=35551, storageInfo=lv=-57;cid=testClusterID;nsid=235672705;c=1732297064796), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:37:46,985 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1
2024-11-22T17:37:46,989 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/zookeeper_0, clientPort=62481, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:37:46,990 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62481
2024-11-22T17:37:46,991 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:46,993 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:37:47,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:37:47,004 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59 with version=8
2024-11-22T17:37:47,004 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:37:47,006 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:37:47,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:37:47,007 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33625
2024-11-22T17:37:47,009 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33625 connecting to ZooKeeper ensemble=127.0.0.1:62481
2024-11-22T17:37:47,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:336250x0, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:37:47,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33625-0x101639f85980000 connected
2024-11-22T17:37:47,157 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,159 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:37:47,162 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59, hbase.cluster.distributed=false
2024-11-22T17:37:47,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:37:47,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33625
2024-11-22T17:37:47,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33625
2024-11-22T17:37:47,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33625
2024-11-22T17:37:47,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33625
2024-11-22T17:37:47,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33625
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:37:47,183 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:37:47,184 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40173
2024-11-22T17:37:47,186 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40173 connecting to ZooKeeper ensemble=127.0.0.1:62481
2024-11-22T17:37:47,186 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,188 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401730x0, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:37:47,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40173-0x101639f85980001 connected
2024-11-22T17:37:47,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:37:47,199 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:37:47,204 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:37:47,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:37:47,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:37:47,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40173
2024-11-22T17:37:47,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40173
2024-11-22T17:37:47,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40173
2024-11-22T17:37:47,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40173
2024-11-22T17:37:47,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40173
2024-11-22T17:37:47,219 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:33625
2024-11-22T17:37:47,219 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:37:47,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:37:47,230 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:37:47,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,241 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:37:47,241 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,33625,1732297067006 from backup master directory
2024-11-22T17:37:47,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:37:47,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:37:47,251 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:37:47,251 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,256 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/hbase.id] with ID: 0a80ceea-3363-4602-8020-2c8f22abc7e9
2024-11-22T17:37:47,256 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/.tmp/hbase.id
2024-11-22T17:37:47,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:37:47,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:37:47,263 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/.tmp/hbase.id]:[hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/hbase.id]
2024-11-22T17:37:47,276 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:47,276 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:37:47,278 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-22T17:37:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:37:47,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:37:47,300 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:37:47,301 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:37:47,302 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:37:47,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:37:47,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:37:47,309 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store
2024-11-22T17:37:47,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:37:47,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:37:47,316 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:47,316 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:37:47,316 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297067316Disabling compacts and flushes for region at 1732297067316Disabling writes for close at 1732297067316Writing region close event to WAL at 1732297067316Closed at 1732297067316
2024-11-22T17:37:47,317 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/.initializing
2024-11-22T17:37:47,317 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/WALs/e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,339 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C33625%2C1732297067006, suffix=, logDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/WALs/e61df7af61d2,33625,1732297067006, archiveDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/oldWALs, maxLogs=10
2024-11-22T17:37:47,339 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33625%2C1732297067006.1732297067339
2024-11-22T17:37:47,344 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/WALs/e61df7af61d2,33625,1732297067006/e61df7af61d2%2C33625%2C1732297067006.1732297067339
2024-11-22T17:37:47,345 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40237:40237),(127.0.0.1/127.0.0.1:33497:33497)]
2024-11-22T17:37:47,346 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:37:47,346 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:47,346 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,346 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,348 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:37:47,350 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:37:47,350 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,350 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:47,350 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:37:47,351 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:37:47,351 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,352 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:37:47,352 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:37:47,353 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:37:47,353 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,353 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:37:47,354 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:37:47,355 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:37:47,355 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,355 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:37:47,355 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,356 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,356 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,358 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,358 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,358 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:37:47,359 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:37:47,365 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:37:47,365 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807273, jitterRate=0.02650076150894165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:37:47,366 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732297067346Initializing all the Stores at 1732297067347 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297067347Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297067348 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297067348Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297067348Cleaning up temporary data from old regions at 1732297067358 (+10 ms)Region opened successfully at 1732297067366 (+8 ms)
2024-11-22T17:37:47,366 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:37:47,369 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e11941a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:37:47,370 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:37:47,370 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:37:47,371 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:37:47,371 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:37:47,371 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:37:47,372 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:37:47,372 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:37:47,374 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:37:47,375 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:37:47,384 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:37:47,385 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:37:47,385 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:37:47,395 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:37:47,395 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:37:47,396 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:37:47,409 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:37:47,410 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:37:47,419 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:37:47,422 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:37:47,430 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:37:47,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:37:47,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:37:47,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,441 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,33625,1732297067006, sessionid=0x101639f85980000, setting cluster-up flag (Was=false)
2024-11-22T17:37:47,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,493 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:37:47,494 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:47,546 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:37:47,546 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,33625,1732297067006
2024-11-22T17:37:47,548 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:37:47,549 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:37:47,549 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:37:47,550 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:37:47,550 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,33625,1732297067006
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:37:47,551 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732297097552
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:37:47,552 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:37:47,553 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297067553,5,FailOnTimeoutGroup]
2024-11-22T17:37:47,553 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297067553,5,FailOnTimeoutGroup]
2024-11-22T17:37:47,553 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,554 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:37:47,554 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,554 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,554 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:37:47,555 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:37:47,555 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,556 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:37:47,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:37:47,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:37:47,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:47,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:47,562 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:37:47,563 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59
2024-11-22T17:37:47,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:37:47,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:37:47,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:47,576 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:37:47,577 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:37:47,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,578 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:47,578 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:37:47,579 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:37:47,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,579 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:47,579 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:37:47,581 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:37:47,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,581 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:47,581 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:37:47,583 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:37:47,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:47,583 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:47,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:37:47,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740
2024-11-22T17:37:47,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740
2024-11-22T17:37:47,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:37:47,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:37:47,586 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:37:47,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:37:47,589 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:37:47,590 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705034, jitterRate=-0.10350413620471954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:37:47,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732297067574Initializing all the Stores at 1732297067575 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297067575Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297067576 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297067576Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297067576Cleaning up temporary data from old regions at 1732297067586 (+10 ms)Region opened successfully at 1732297067590 (+4 ms)
2024-11-22T17:37:47,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:37:47,590 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:37:47,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:37:47,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:37:47,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:37:47,591 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:37:47,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297067590Disabling compacts and flushes for region at 1732297067590Disabling writes for close at 1732297067591 (+1 ms)Writing region close event to WAL at 1732297067591Closed at 1732297067591
2024-11-22T17:37:47,592 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:37:47,592 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:37:47,592 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:37:47,594 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:37:47,595 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:37:47,609 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(746): ClusterId : 0a80ceea-3363-4602-8020-2c8f22abc7e9
2024-11-22T17:37:47,609 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:37:47,620 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:37:47,620 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:37:47,630 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:37:47,631 DEBUG [RS:0;e61df7af61d2:40173 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c54376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:37:47,649 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:40173
2024-11-22T17:37:47,649 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:37:47,649 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:37:47,649 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:37:47,650 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,33625,1732297067006 with port=40173, startcode=1732297067182
2024-11-22T17:37:47,650 DEBUG [RS:0;e61df7af61d2:40173 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:37:47,652 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53513, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:37:47,652 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33625 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,652 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33625 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,654 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59
2024-11-22T17:37:47,654 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42755
2024-11-22T17:37:47,654 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:37:47,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:37:47,662 DEBUG [RS:0;e61df7af61d2:40173 {}] zookeeper.ZKUtil(111): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,662 WARN  [RS:0;e61df7af61d2:40173 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:37:47,662 INFO  [RS:0;e61df7af61d2:40173 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:37:47,662 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,662 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,40173,1732297067182]
2024-11-22T17:37:47,666 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:37:47,668 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:37:47,668 INFO  [RS:0;e61df7af61d2:40173 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:37:47,668 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,668 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:37:47,669 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:37:47,669 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:37:47,670 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:37:47,671 DEBUG [RS:0;e61df7af61d2:40173 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,671 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,40173,1732297067182-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:37:47,690 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:37:47,690 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,40173,1732297067182-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,691 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,691 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.Replication(171): e61df7af61d2,40173,1732297067182 started
2024-11-22T17:37:47,707 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:47,707 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,40173,1732297067182, RpcServer on e61df7af61d2/172.17.0.2:40173, sessionid=0x101639f85980001
2024-11-22T17:37:47,707 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:37:47,707 DEBUG [RS:0;e61df7af61d2:40173 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,707 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,40173,1732297067182'
2024-11-22T17:37:47,707 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:37:47,707 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,40173,1732297067182'
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:37:47,708 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:37:47,709 DEBUG [RS:0;e61df7af61d2:40173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:37:47,709 INFO  [RS:0;e61df7af61d2:40173 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:37:47,709 INFO  [RS:0;e61df7af61d2:40173 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:37:47,745 WARN  [e61df7af61d2:33625 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:37:47,811 INFO  [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C40173%2C1732297067182, suffix=, logDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182, archiveDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs, maxLogs=32
2024-11-22T17:37:47,812 INFO  [RS:0;e61df7af61d2:40173 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C40173%2C1732297067182.1732297067811
2024-11-22T17:37:47,819 INFO  [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297067811
2024-11-22T17:37:47,820 DEBUG [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40237:40237),(127.0.0.1/127.0.0.1:33497:33497)]
2024-11-22T17:37:47,995 DEBUG [e61df7af61d2:33625 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:37:47,996 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:47,997 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,40173,1732297067182, state=OPENING
2024-11-22T17:37:48,009 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:37:48,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:48,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:37:48,230 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:37:48,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:37:48,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:37:48,231 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,40173,1732297067182}]
2024-11-22T17:37:48,384 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:37:48,387 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54389, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:37:48,391 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:37:48,392 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:37:48,394 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C40173%2C1732297067182.meta, suffix=.meta, logDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182, archiveDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs, maxLogs=32
2024-11-22T17:37:48,395 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C40173%2C1732297067182.meta.1732297068394.meta
2024-11-22T17:37:48,401 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.meta.1732297068394.meta
2024-11-22T17:37:48,402 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33497:33497),(127.0.0.1/127.0.0.1:40237:40237)]
2024-11-22T17:37:48,403 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:37:48,403 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:37:48,403 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:37:48,403 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:37:48,403 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:37:48,403 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:48,404 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:37:48,404 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:37:48,405 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:37:48,406 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:37:48,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:48,407 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:37:48,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:37:48,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:48,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:37:48,408 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:37:48,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:48,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:37:48,410 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:37:48,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,410 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:37:48,410 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:37:48,411 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740
2024-11-22T17:37:48,412 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740
2024-11-22T17:37:48,413 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:37:48,413 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:37:48,413 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:37:48,415 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:37:48,415 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805669, jitterRate=0.024461418390274048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:37:48,415 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:37:48,416 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732297068404Writing region info on filesystem at 1732297068404Initializing all the Stores at 1732297068405 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297068405Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297068405Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297068405Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297068405Cleaning up temporary data from old regions at 1732297068413 (+8 ms)Running coprocessor post-open hooks at 1732297068415 (+2 ms)Region opened successfully at 1732297068416 (+1 ms)
2024-11-22T17:37:48,417 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732297068384
2024-11-22T17:37:48,419 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:37:48,419 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:37:48,420 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:48,421 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,40173,1732297067182, state=OPEN
2024-11-22T17:37:48,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:37:48,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:37:48,459 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:48,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:37:48,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:37:48,461 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:37:48,461 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,40173,1732297067182 in 228 msec
2024-11-22T17:37:48,464 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:37:48,464 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 869 msec
2024-11-22T17:37:48,465 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:37:48,465 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:37:48,466 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:37:48,466 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,40173,1732297067182, seqNum=-1]
2024-11-22T17:37:48,466 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:37:48,467 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47459, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:37:48,472 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 923 msec
2024-11-22T17:37:48,473 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732297068472, completionTime=-1
2024-11-22T17:37:48,473 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:37:48,473 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732297128475
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297188475
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:33625, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,475 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,477 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.228sec
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:37:48,479 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:37:48,482 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:37:48,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:37:48,482 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33625,1732297067006-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:37:48,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ac0e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:37:48,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,33625,-1 for getting cluster id
2024-11-22T17:37:48,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:37:48,512 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a80ceea-3363-4602-8020-2c8f22abc7e9'
2024-11-22T17:37:48,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:37:48,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a80ceea-3363-4602-8020-2c8f22abc7e9"

2024-11-22T17:37:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533a12fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:37:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,33625,-1]
2024-11-22T17:37:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:37:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:37:48,514 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:37:48,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4122179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:37:48,515 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:37:48,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,40173,1732297067182, seqNum=-1]
2024-11-22T17:37:48,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:37:48,518 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:37:48,519 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,33625,1732297067006
2024-11-22T17:37:48,520 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:37:48,522 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:37:48,522 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-22T17:37:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e61df7af61d2,33625,1732297067006
2024-11-22T17:37:48,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a957c1b
2024-11-22T17:37:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-22T17:37:48,525 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-22T17:37:48,525 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-22T17:37:48,526 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-22T17:37:48,526 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:37:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-22T17:37:48,528 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-22T17:37:48,528 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,528 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-22T17:37:48,529 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-22T17:37:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:37:48,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741835_1011 (size=381)
2024-11-22T17:37:48,536 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741835_1011 (size=381)
2024-11-22T17:37:48,537 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce559e202d2ab9fd88de1fd31c2bc36b, NAME => 'TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59
2024-11-22T17:37:48,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741836_1012 (size=64)
2024-11-22T17:37:48,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741836_1012 (size=64)
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing ce559e202d2ab9fd88de1fd31c2bc36b, disabling compactions & flushes
2024-11-22T17:37:48,545 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. after waiting 0 ms
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,545 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,545 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce559e202d2ab9fd88de1fd31c2bc36b:
Waiting for close lock at 1732297068545Disabling compacts and flushes for region at 1732297068545Disabling writes for close at 1732297068545Writing region close event to WAL at 1732297068545Closed at 1732297068545
2024-11-22T17:37:48,546 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-22T17:37:48,547 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732297068546"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732297068546"}]},"ts":"1732297068546"}
2024-11-22T17:37:48,549 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-22T17:37:48,550 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-22T17:37:48,551 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732297068550"}]},"ts":"1732297068550"}
2024-11-22T17:37:48,553 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-22T17:37:48,554 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, ASSIGN}]
2024-11-22T17:37:48,555 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, ASSIGN
2024-11-22T17:37:48,556 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, ASSIGN; state=OFFLINE, location=e61df7af61d2,40173,1732297067182; forceNewPlan=false, retain=false
2024-11-22T17:37:48,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:48,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:48,707 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce559e202d2ab9fd88de1fd31c2bc36b, regionState=OPENING, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:48,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, ASSIGN because future has completed
2024-11-22T17:37:48,710 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182}]
2024-11-22T17:37:48,869 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,869 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ce559e202d2ab9fd88de1fd31c2bc36b, NAME => 'TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:37:48,870 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,870 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:37:48,870 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,870 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,872 INFO  [StoreOpener-ce559e202d2ab9fd88de1fd31c2bc36b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ce559e202d2ab9fd88de1fd31c2bc36b 
2024-11-22T17:37:48,875 INFO  [StoreOpener-ce559e202d2ab9fd88de1fd31c2bc36b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce559e202d2ab9fd88de1fd31c2bc36b columnFamilyName info
2024-11-22T17:37:48,875 DEBUG [StoreOpener-ce559e202d2ab9fd88de1fd31c2bc36b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:37:48,875 INFO  [StoreOpener-ce559e202d2ab9fd88de1fd31c2bc36b-1 {}] regionserver.HStore(327): Store=ce559e202d2ab9fd88de1fd31c2bc36b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:37:48,876 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,876 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,877 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,877 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,877 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,879 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,882 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:37:48,882 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ce559e202d2ab9fd88de1fd31c2bc36b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844256, jitterRate=0.07352708280086517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:37:48,882 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:48,883 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ce559e202d2ab9fd88de1fd31c2bc36b:
Running coprocessor pre-open hook at 1732297068870Writing region info on filesystem at 1732297068870Initializing all the Stores at 1732297068872 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297068872Cleaning up temporary data from old regions at 1732297068877 (+5 ms)Running coprocessor post-open hooks at 1732297068882 (+5 ms)Region opened successfully at 1732297068883 (+1 ms)
2024-11-22T17:37:48,884 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., pid=6, masterSystemTime=1732297068864
2024-11-22T17:37:48,887 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,887 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:48,888 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ce559e202d2ab9fd88de1fd31c2bc36b, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:37:48,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182 because future has completed
2024-11-22T17:37:48,896 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-22T17:37:48,896 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182 in 183 msec
2024-11-22T17:37:48,898 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-22T17:37:48,898 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, ASSIGN in 342 msec
2024-11-22T17:37:48,899 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-22T17:37:48,899 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732297068899"}]},"ts":"1732297068899"}
2024-11-22T17:37:48,902 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-22T17:37:48,903 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-22T17:37:48,905 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 377 msec
2024-11-22T17:37:49,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,153 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,155 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,177 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,177 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,177 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,181 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,181 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,182 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,184 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:49,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:49,689 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:37:49,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,690 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,691 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,691 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,691 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,716 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,720 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,720 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:49,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:50,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:50,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:51,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-22T17:37:51,205 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T17:37:51,206 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-22T17:37:51,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:51,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:52,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:52,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:53,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:53,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:53,666 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-22T17:37:53,669 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-22T17:37:54,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:54,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:55,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:55,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:56,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:56,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:56,711 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:37:56,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,716 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,717 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,743 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,743 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,743 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:56,750 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:37:57,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:57,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33625 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-22T17:37:58,563 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-22T17:37:58,563 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-22T17:37:58,566 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-22T17:37:58,566 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:37:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2]
2024-11-22T17:37:58,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:58,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:58,585 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:37:58,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/aea83367af42494b841ab080745094dd is 1080, key is row0001/info:/1732297078571/Put/seqid=0
2024-11-22T17:37:58,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741837_1013 (size=12509)
2024-11-22T17:37:58,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741837_1013 (size=12509)
2024-11-22T17:37:58,609 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/aea83367af42494b841ab080745094dd
2024-11-22T17:37:58,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/aea83367af42494b841ab080745094dd as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd
2024-11-22T17:37:58,623 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd, entries=7, sequenceid=11, filesize=12.2 K
2024-11-22T17:37:58,624 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for ce559e202d2ab9fd88de1fd31c2bc36b in 38ms, sequenceid=11, compaction requested=false
2024-11-22T17:37:58,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:37:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:37:58,625 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-22T17:37:58,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/18e15977a8ac41ac8f41fed86c25b8f1 is 1080, key is row0008/info:/1732297078586/Put/seqid=0
2024-11-22T17:37:58,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741838_1014 (size=25453)
2024-11-22T17:37:58,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741838_1014 (size=25453)
2024-11-22T17:37:58,638 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/18e15977a8ac41ac8f41fed86c25b8f1
2024-11-22T17:37:58,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/18e15977a8ac41ac8f41fed86c25b8f1 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1
2024-11-22T17:37:58,652 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1, entries=19, sequenceid=33, filesize=24.9 K
2024-11-22T17:37:58,654 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for ce559e202d2ab9fd88de1fd31c2bc36b in 28ms, sequenceid=33, compaction requested=false
2024-11-22T17:37:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:37:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=37.1 K, sizeToCheck=16.0 K
2024-11-22T17:37:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:37:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1 because midkey is the same as first or last row
2024-11-22T17:37:59,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:37:59,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:00,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:00,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:00,641 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:00,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/2836b5658f5c4e6fba6be1cfe06db58f is 1080, key is row0027/info:/1732297078626/Put/seqid=0
2024-11-22T17:38:00,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741839_1015 (size=12509)
2024-11-22T17:38:00,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741839_1015 (size=12509)
2024-11-22T17:38:00,667 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/2836b5658f5c4e6fba6be1cfe06db58f
2024-11-22T17:38:00,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/2836b5658f5c4e6fba6be1cfe06db58f as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f
2024-11-22T17:38:00,690 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f, entries=7, sequenceid=43, filesize=12.2 K
2024-11-22T17:38:00,691 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for ce559e202d2ab9fd88de1fd31c2bc36b in 50ms, sequenceid=43, compaction requested=true
2024-11-22T17:38:00,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:38:00,691 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-22T17:38:00,691 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:00,691 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1 because midkey is the same as first or last row
2024-11-22T17:38:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:00,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce559e202d2ab9fd88de1fd31c2bc36b:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:00,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:00,693 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-22T17:38:00,693 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:00,696 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:00,696 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): ce559e202d2ab9fd88de1fd31c2bc36b/info is initiating minor compaction (all files)
2024-11-22T17:38:00,696 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce559e202d2ab9fd88de1fd31c2bc36b/info in TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:00,696 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp, totalSize=49.3 K
2024-11-22T17:38:00,697 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea83367af42494b841ab080745094dd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732297078571
2024-11-22T17:38:00,698 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e15977a8ac41ac8f41fed86c25b8f1, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732297078586
2024-11-22T17:38:00,699 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2836b5658f5c4e6fba6be1cfe06db58f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732297078626
2024-11-22T17:38:00,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/4803bd953e19403da3b25ec015922e27 is 1080, key is row0034/info:/1732297080643/Put/seqid=0
2024-11-22T17:38:00,716 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce559e202d2ab9fd88de1fd31c2bc36b#info#compaction#60 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:00,716 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/8c1a3474374746a296f3fd9e21448c99 is 1080, key is row0001/info:/1732297078571/Put/seqid=0
2024-11-22T17:38:00,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741841_1017 (size=40670)
2024-11-22T17:38:00,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741841_1017 (size=40670)
2024-11-22T17:38:00,735 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-22T17:38:00,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741840_1016 (size=21141)
2024-11-22T17:38:00,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741840_1016 (size=21141)
2024-11-22T17:38:00,736 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/8c1a3474374746a296f3fd9e21448c99 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99
2024-11-22T17:38:00,737 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/4803bd953e19403da3b25ec015922e27
2024-11-22T17:38:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36224 deadline: 1732297090734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:00,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/4803bd953e19403da3b25ec015922e27 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27
2024-11-22T17:38:00,746 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce559e202d2ab9fd88de1fd31c2bc36b/info of ce559e202d2ab9fd88de1fd31c2bc36b into 8c1a3474374746a296f3fd9e21448c99(size=39.7 K), total size for store is 39.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:00,746 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:38:00,746 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., storeName=ce559e202d2ab9fd88de1fd31c2bc36b/info, priority=13, startTime=1732297080691; duration=0sec
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.7 K, sizeToCheck=16.0 K
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 because midkey is the same as first or last row
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.7 K, sizeToCheck=16.0 K
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 because midkey is the same as first or last row
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.7 K, sizeToCheck=16.0 K
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 because midkey is the same as first or last row
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:00,747 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce559e202d2ab9fd88de1fd31c2bc36b:info
2024-11-22T17:38:00,750 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27, entries=15, sequenceid=61, filesize=20.6 K
2024-11-22T17:38:00,751 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for ce559e202d2ab9fd88de1fd31c2bc36b in 59ms, sequenceid=61, compaction requested=false
2024-11-22T17:38:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:38:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.4 K, sizeToCheck=16.0 K
2024-11-22T17:38:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:00,751 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 because midkey is the same as first or last row
2024-11-22T17:38:00,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T17:38:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T17:38:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 because the exception is null or not the one we care about
2024-11-22T17:38:01,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:01,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:02,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:02,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:03,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:03,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:04,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:04,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:05,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:05,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:06,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:06,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:07,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:07,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:08,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:08,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:09,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:09,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:10,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:10,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:10,817 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-22T17:38:10,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/cb50330615494c2d84e02d083b799a99 is 1080, key is row0049/info:/1732297080694/Put/seqid=0
2024-11-22T17:38:10,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741842_1018 (size=21141)
2024-11-22T17:38:10,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741842_1018 (size=21141)
2024-11-22T17:38:10,832 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/cb50330615494c2d84e02d083b799a99
2024-11-22T17:38:10,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/cb50330615494c2d84e02d083b799a99 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99
2024-11-22T17:38:10,846 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99, entries=15, sequenceid=80, filesize=20.6 K
2024-11-22T17:38:10,847 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=1.05 KB/1076 for ce559e202d2ab9fd88de1fd31c2bc36b in 29ms, sequenceid=80, compaction requested=true
2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.0 K, sizeToCheck=16.0 K
2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 because midkey is the same as first or last row
2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ce559e202d2ab9fd88de1fd31c2bc36b:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:10,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:10,847 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:10,848 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 82952 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:10,848 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): ce559e202d2ab9fd88de1fd31c2bc36b/info is initiating minor compaction (all files)
2024-11-22T17:38:10,848 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ce559e202d2ab9fd88de1fd31c2bc36b/info in TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:10,848 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp, totalSize=81.0 K
2024-11-22T17:38:10,849 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1a3474374746a296f3fd9e21448c99, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732297078571
2024-11-22T17:38:10,849 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4803bd953e19403da3b25ec015922e27, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732297080643
2024-11-22T17:38:10,849 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb50330615494c2d84e02d083b799a99, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732297080694
2024-11-22T17:38:10,860 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ce559e202d2ab9fd88de1fd31c2bc36b#info#compaction#62 average throughput is 32.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:10,860 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/a48a4f595e014b0f94db7ac4f3e8cec5 is 1080, key is row0001/info:/1732297078571/Put/seqid=0
2024-11-22T17:38:10,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741843_1019 (size=73224)
2024-11-22T17:38:10,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741843_1019 (size=73224)
2024-11-22T17:38:10,871 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/a48a4f595e014b0f94db7ac4f3e8cec5 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5
2024-11-22T17:38:10,876 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ce559e202d2ab9fd88de1fd31c2bc36b/info of ce559e202d2ab9fd88de1fd31c2bc36b into a48a4f595e014b0f94db7ac4f3e8cec5(size=71.5 K), total size for store is 71.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ce559e202d2ab9fd88de1fd31c2bc36b:

2024-11-22T17:38:10,876 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., storeName=ce559e202d2ab9fd88de1fd31c2bc36b/info, priority=13, startTime=1732297090847; duration=0sec
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.5 K, sizeToCheck=16.0 K
2024-11-22T17:38:10,876 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-22T17:38:10,877 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:10,877 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:10,877 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ce559e202d2ab9fd88de1fd31c2bc36b:info
2024-11-22T17:38:10,878 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33625 {}] assignment.AssignmentManager(1363): Split request from e61df7af61d2,40173,1732297067182, parent={ENCODED => ce559e202d2ab9fd88de1fd31c2bc36b, NAME => 'TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-22T17:38:10,883 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33625 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:10,886 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33625 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ce559e202d2ab9fd88de1fd31c2bc36b, daughterA=a8048cdbcee3d4f4fb7b457f3ee6e4db, daughterB=4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:10,887 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ce559e202d2ab9fd88de1fd31c2bc36b, daughterA=a8048cdbcee3d4f4fb7b457f3ee6e4db, daughterB=4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:10,887 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ce559e202d2ab9fd88de1fd31c2bc36b, daughterA=a8048cdbcee3d4f4fb7b457f3ee6e4db, daughterB=4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:10,888 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ce559e202d2ab9fd88de1fd31c2bc36b, daughterA=a8048cdbcee3d4f4fb7b457f3ee6e4db, daughterB=4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:10,894 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, UNASSIGN}]
2024-11-22T17:38:10,895 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, UNASSIGN
2024-11-22T17:38:10,896 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ce559e202d2ab9fd88de1fd31c2bc36b, regionState=CLOSING, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:10,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, UNASSIGN because future has completed
2024-11-22T17:38:10,899 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-22T17:38:10,899 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182}]
2024-11-22T17:38:11,059 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,060 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-22T17:38:11,060 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing ce559e202d2ab9fd88de1fd31c2bc36b, disabling compactions & flushes
2024-11-22T17:38:11,061 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:11,061 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:11,061 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. after waiting 0 ms
2024-11-22T17:38:11,061 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:11,061 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing ce559e202d2ab9fd88de1fd31c2bc36b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-22T17:38:11,065 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/a27c043181fc4db3901f54a0e54e2e1e is 1080, key is row0064/info:/1732297090820/Put/seqid=0
2024-11-22T17:38:11,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741844_1020 (size=6033)
2024-11-22T17:38:11,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741844_1020 (size=6033)
2024-11-22T17:38:11,071 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/a27c043181fc4db3901f54a0e54e2e1e
2024-11-22T17:38:11,075 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/.tmp/info/a27c043181fc4db3901f54a0e54e2e1e as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a27c043181fc4db3901f54a0e54e2e1e
2024-11-22T17:38:11,081 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a27c043181fc4db3901f54a0e54e2e1e, entries=1, sequenceid=85, filesize=5.9 K
2024-11-22T17:38:11,082 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce559e202d2ab9fd88de1fd31c2bc36b in 21ms, sequenceid=85, compaction requested=false
2024-11-22T17:38:11,082 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99] to archive
2024-11-22T17:38:11,083 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:38:11,085 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/aea83367af42494b841ab080745094dd
2024-11-22T17:38:11,086 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/18e15977a8ac41ac8f41fed86c25b8f1
2024-11-22T17:38:11,087 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/8c1a3474374746a296f3fd9e21448c99
2024-11-22T17:38:11,088 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/2836b5658f5c4e6fba6be1cfe06db58f
2024-11-22T17:38:11,089 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/4803bd953e19403da3b25ec015922e27
2024-11-22T17:38:11,091 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/cb50330615494c2d84e02d083b799a99
2024-11-22T17:38:11,097 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=1
2024-11-22T17:38:11,098 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.
2024-11-22T17:38:11,098 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for ce559e202d2ab9fd88de1fd31c2bc36b:
Waiting for close lock at 1732297091060Running coprocessor pre-close hooks at 1732297091060Disabling compacts and flushes for region at 1732297091060Disabling writes for close at 1732297091061 (+1 ms)Obtaining lock to block concurrent updates at 1732297091061Preparing flush snapshotting stores in ce559e202d2ab9fd88de1fd31c2bc36b at 1732297091061Finished memstore snapshotting TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732297091061Flushing stores of TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. at 1732297091062 (+1 ms)Flushing ce559e202d2ab9fd88de1fd31c2bc36b/info: creating writer at 1732297091062Flushing ce559e202d2ab9fd88de1fd31c2bc36b/info: appending metadata at 1732297091065 (+3 ms)Flushing ce559e202d2ab9fd88de1fd31c2bc36b/info: closing flushed file at 1732297091065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3207a04c: reopening flushed file at 1732297091075 (+10 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ce559e202d2ab9fd88de1fd31c2bc36b in 21ms, sequenceid=85, compaction requested=false at 1732297091082 (+7 ms)Writing region close event to WAL at 1732297091093 (+11 ms)Running coprocessor post-close hooks at 1732297091098 (+5 ms)Closed at 1732297091098
2024-11-22T17:38:11,101 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,102 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ce559e202d2ab9fd88de1fd31c2bc36b, regionState=CLOSED
2024-11-22T17:38:11,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182 because future has completed
2024-11-22T17:38:11,107 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-22T17:38:11,107 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure ce559e202d2ab9fd88de1fd31c2bc36b, server=e61df7af61d2,40173,1732297067182 in 206 msec
2024-11-22T17:38:11,110 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-22T17:38:11,110 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ce559e202d2ab9fd88de1fd31c2bc36b, UNASSIGN in 213 msec
2024-11-22T17:38:11,117 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:11,120 INFO  [PEWorker-1 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 2 storefiles, region=ce559e202d2ab9fd88de1fd31c2bc36b, threads=2
2024-11-22T17:38:11,122 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5 for region: ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,122 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a27c043181fc4db3901f54a0e54e2e1e for region: ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,131 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a27c043181fc4db3901f54a0e54e2e1e, top=true
2024-11-22T17:38:11,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741845_1021 (size=27)
2024-11-22T17:38:11,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741845_1021 (size=27)
2024-11-22T17:38:11,144 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e for child: 4b9b5859ba38afad0e83fdf9d5d567af, parent: ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,144 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a27c043181fc4db3901f54a0e54e2e1e for region: ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741846_1022 (size=27)
2024-11-22T17:38:11,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741846_1022 (size=27)
2024-11-22T17:38:11,154 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5 for region: ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:11,157 DEBUG [PEWorker-1 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region ce559e202d2ab9fd88de1fd31c2bc36b Daughter A: [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b] storefiles, Daughter B: [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b] storefiles.
2024-11-22T17:38:11,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741847_1023 (size=71)
2024-11-22T17:38:11,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741847_1023 (size=71)
2024-11-22T17:38:11,171 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:11,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741848_1024 (size=71)
2024-11-22T17:38:11,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741848_1024 (size=71)
2024-11-22T17:38:11,193 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:11,203 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=-1
2024-11-22T17:38:11,205 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=-1
2024-11-22T17:38:11,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-22T17:38:11,208 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732297091207"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732297091207"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732297091207"}]},"ts":"1732297091207"}
2024-11-22T17:38:11,208 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732297091207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732297091207"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732297091207"}]},"ts":"1732297091207"}
2024-11-22T17:38:11,208 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732297091207"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732297091207"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732297091207"}]},"ts":"1732297091207"}
2024-11-22T17:38:11,226 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a8048cdbcee3d4f4fb7b457f3ee6e4db, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4b9b5859ba38afad0e83fdf9d5d567af, ASSIGN}]
2024-11-22T17:38:11,228 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a8048cdbcee3d4f4fb7b457f3ee6e4db, ASSIGN
2024-11-22T17:38:11,228 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4b9b5859ba38afad0e83fdf9d5d567af, ASSIGN
2024-11-22T17:38:11,229 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a8048cdbcee3d4f4fb7b457f3ee6e4db, ASSIGN; state=SPLITTING_NEW, location=e61df7af61d2,40173,1732297067182; forceNewPlan=false, retain=false
2024-11-22T17:38:11,229 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4b9b5859ba38afad0e83fdf9d5d567af, ASSIGN; state=SPLITTING_NEW, location=e61df7af61d2,40173,1732297067182; forceNewPlan=false, retain=false
2024-11-22T17:38:11,379 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a8048cdbcee3d4f4fb7b457f3ee6e4db, regionState=OPENING, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:11,379 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4b9b5859ba38afad0e83fdf9d5d567af, regionState=OPENING, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:11,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a8048cdbcee3d4f4fb7b457f3ee6e4db, ASSIGN because future has completed
2024-11-22T17:38:11,383 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8048cdbcee3d4f4fb7b457f3ee6e4db, server=e61df7af61d2,40173,1732297067182}]
2024-11-22T17:38:11,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4b9b5859ba38afad0e83fdf9d5d567af, ASSIGN because future has completed
2024-11-22T17:38:11,384 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9b5859ba38afad0e83fdf9d5d567af, server=e61df7af61d2,40173,1732297067182}]
2024-11-22T17:38:11,540 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:11,540 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => a8048cdbcee3d4f4fb7b457f3ee6e4db, NAME => 'TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-22T17:38:11,541 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,541 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:11,541 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,541 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,543 INFO  [StoreOpener-a8048cdbcee3d4f4fb7b457f3ee6e4db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a8048cdbcee3d4f4fb7b457f3ee6e4db 
2024-11-22T17:38:11,544 INFO  [StoreOpener-a8048cdbcee3d4f4fb7b457f3ee6e4db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a8048cdbcee3d4f4fb7b457f3ee6e4db columnFamilyName info
2024-11-22T17:38:11,544 DEBUG [StoreOpener-a8048cdbcee3d4f4fb7b457f3ee6e4db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:11,559 DEBUG [StoreOpener-a8048cdbcee3d4f4fb7b457f3ee6e4db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-bottom
2024-11-22T17:38:11,560 INFO  [StoreOpener-a8048cdbcee3d4f4fb7b457f3ee6e4db-1 {}] regionserver.HStore(327): Store=a8048cdbcee3d4f4fb7b457f3ee6e4db/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:38:11,560 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,561 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,563 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,563 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,563 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,566 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,567 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened a8048cdbcee3d4f4fb7b457f3ee6e4db; next sequenceid=89; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705656, jitterRate=-0.10271245241165161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:38:11,567 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:11,568 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for a8048cdbcee3d4f4fb7b457f3ee6e4db:
Running coprocessor pre-open hook at 1732297091541Writing region info on filesystem at 1732297091541Initializing all the Stores at 1732297091542 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297091543 (+1 ms)Cleaning up temporary data from old regions at 1732297091563 (+20 ms)Running coprocessor post-open hooks at 1732297091567 (+4 ms)Region opened successfully at 1732297091568 (+1 ms)
2024-11-22T17:38:11,569 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db., pid=12, masterSystemTime=1732297091536
2024-11-22T17:38:11,569 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store a8048cdbcee3d4f4fb7b457f3ee6e4db:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:11,569 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:11,569 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-22T17:38:11,570 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:11,570 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): a8048cdbcee3d4f4fb7b457f3ee6e4db/info is initiating minor compaction (all files)
2024-11-22T17:38:11,570 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a8048cdbcee3d4f4fb7b457f3ee6e4db/info in TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:11,571 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-bottom] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/.tmp, totalSize=71.5 K
2024-11-22T17:38:11,572 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b, keycount=31, bloomtype=ROW, size=71.5 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732297078571
2024-11-22T17:38:11,573 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:11,573 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:11,573 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:11,573 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b9b5859ba38afad0e83fdf9d5d567af, NAME => 'TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-22T17:38:11,574 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,574 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:11,574 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,574 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,574 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a8048cdbcee3d4f4fb7b457f3ee6e4db, regionState=OPEN, openSeqNum=89, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:11,575 INFO  [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4b9b5859ba38afad0e83fdf9d5d567af 
2024-11-22T17:38:11,576 INFO  [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9b5859ba38afad0e83fdf9d5d567af columnFamilyName info
2024-11-22T17:38:11,576 DEBUG [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:11,576 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-22T17:38:11,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-22T17:38:11,577 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-22T17:38:11,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a8048cdbcee3d4f4fb7b457f3ee6e4db, server=e61df7af61d2,40173,1732297067182 because future has completed
2024-11-22T17:38:11,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:11,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:11,584 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-22T17:38:11,584 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a8048cdbcee3d4f4fb7b457f3ee6e4db, server=e61df7af61d2,40173,1732297067182 in 198 msec
2024-11-22T17:38:11,586 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a8048cdbcee3d4f4fb7b457f3ee6e4db, ASSIGN in 358 msec
2024-11-22T17:38:11,586 DEBUG [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e
2024-11-22T17:38:11,590 DEBUG [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-top
2024-11-22T17:38:11,590 INFO  [StoreOpener-4b9b5859ba38afad0e83fdf9d5d567af-1 {}] regionserver.HStore(327): Store=4b9b5859ba38afad0e83fdf9d5d567af/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:38:11,591 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,591 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,592 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,592 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,593 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,594 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,594 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 4b9b5859ba38afad0e83fdf9d5d567af; next sequenceid=89; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=746965, jitterRate=-0.05018545687198639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-22T17:38:11,594 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:11,595 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 4b9b5859ba38afad0e83fdf9d5d567af:
Running coprocessor pre-open hook at 1732297091574Writing region info on filesystem at 1732297091574Initializing all the Stores at 1732297091575 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297091575Cleaning up temporary data from old regions at 1732297091593 (+18 ms)Running coprocessor post-open hooks at 1732297091594 (+1 ms)Region opened successfully at 1732297091595 (+1 ms)
2024-11-22T17:38:11,595 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., pid=13, masterSystemTime=1732297091536
2024-11-22T17:38:11,595 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 2
2024-11-22T17:38:11,595 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:11,595 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking
2024-11-22T17:38:11,596 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:11,596 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:11,596 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:11,597 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-top, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=77.4 K
2024-11-22T17:38:11,597 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] compactions.Compactor(225): Compacting a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b, keycount=31, bloomtype=ROW, size=71.5 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732297078571
2024-11-22T17:38:11,598 DEBUG [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:11,598 INFO  [RS_OPEN_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:11,598 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732297090820
2024-11-22T17:38:11,599 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4b9b5859ba38afad0e83fdf9d5d567af, regionState=OPEN, openSeqNum=89, regionLocation=e61df7af61d2,40173,1732297067182
2024-11-22T17:38:11,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/4ad04e0134fd42068fd316e8870f9886 is 193, key is TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af./info:regioninfo/1732297091379/Put/seqid=0
2024-11-22T17:38:11,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9b5859ba38afad0e83fdf9d5d567af, server=e61df7af61d2,40173,1732297067182 because future has completed
2024-11-22T17:38:11,601 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a8048cdbcee3d4f4fb7b457f3ee6e4db#info#compaction#64 average throughput is 15.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:11,601 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/.tmp/info/0650ff6cd2aa4776835daad72d741f72 is 1080, key is row0001/info:/1732297078571/Put/seqid=0
2024-11-22T17:38:11,605 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-22T17:38:11,605 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b9b5859ba38afad0e83fdf9d5d567af, server=e61df7af61d2,40173,1732297067182 in 218 msec
2024-11-22T17:38:11,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741849_1025 (size=9847)
2024-11-22T17:38:11,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741849_1025 (size=9847)
2024-11-22T17:38:11,606 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/4ad04e0134fd42068fd316e8870f9886
2024-11-22T17:38:11,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741850_1026 (size=70862)
2024-11-22T17:38:11,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741850_1026 (size=70862)
2024-11-22T17:38:11,608 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-22T17:38:11,608 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4b9b5859ba38afad0e83fdf9d5d567af, ASSIGN in 379 msec
2024-11-22T17:38:11,613 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=ce559e202d2ab9fd88de1fd31c2bc36b, daughterA=a8048cdbcee3d4f4fb7b457f3ee6e4db, daughterB=4b9b5859ba38afad0e83fdf9d5d567af in 725 msec
2024-11-22T17:38:11,613 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/.tmp/info/0650ff6cd2aa4776835daad72d741f72 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/0650ff6cd2aa4776835daad72d741f72
2024-11-22T17:38:11,619 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in a8048cdbcee3d4f4fb7b457f3ee6e4db/info of a8048cdbcee3d4f4fb7b457f3ee6e4db into 0650ff6cd2aa4776835daad72d741f72(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:11,619 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a8048cdbcee3d4f4fb7b457f3ee6e4db:

2024-11-22T17:38:11,619 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db., storeName=a8048cdbcee3d4f4fb7b457f3ee6e4db/info, priority=15, startTime=1732297091569; duration=0sec
2024-11-22T17:38:11,619 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:11,619 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a8048cdbcee3d4f4fb7b457f3ee6e4db:info
2024-11-22T17:38:11,622 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#66 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:11,622 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/8c7c97431359450fabe37aee55ef3c14 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:11,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741851_1027 (size=8359)
2024-11-22T17:38:11,629 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741851_1027 (size=8359)
2024-11-22T17:38:11,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/ns/5899073e201748d880c1cade32a11a88 is 43, key is default/ns:d/1732297068468/Put/seqid=0
2024-11-22T17:38:11,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741852_1028 (size=5153)
2024-11-22T17:38:11,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741852_1028 (size=5153)
2024-11-22T17:38:11,635 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/ns/5899073e201748d880c1cade32a11a88
2024-11-22T17:38:11,635 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/8c7c97431359450fabe37aee55ef3c14 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/8c7c97431359450fabe37aee55ef3c14
2024-11-22T17:38:11,640 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 2 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 8c7c97431359450fabe37aee55ef3c14(size=8.2 K), total size for store is 8.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:11,640 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:11,640 INFO  [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=14, startTime=1732297091595; duration=0sec
2024-11-22T17:38:11,641 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:11,641 DEBUG [RS:0;e61df7af61d2:40173-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:11,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/table/a2d427ebc09049d9973c774b71c9af72 is 65, key is TestLogRolling-testLogRolling/table:state/1732297068899/Put/seqid=0
2024-11-22T17:38:11,655 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741853_1029 (size=5340)
2024-11-22T17:38:11,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741853_1029 (size=5340)
2024-11-22T17:38:11,656 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/table/a2d427ebc09049d9973c774b71c9af72
2024-11-22T17:38:11,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/4ad04e0134fd42068fd316e8870f9886 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/info/4ad04e0134fd42068fd316e8870f9886
2024-11-22T17:38:11,665 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/info/4ad04e0134fd42068fd316e8870f9886, entries=30, sequenceid=17, filesize=9.6 K
2024-11-22T17:38:11,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/ns/5899073e201748d880c1cade32a11a88 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/ns/5899073e201748d880c1cade32a11a88
2024-11-22T17:38:11,671 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/ns/5899073e201748d880c1cade32a11a88, entries=2, sequenceid=17, filesize=5.0 K
2024-11-22T17:38:11,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/table/a2d427ebc09049d9973c774b71c9af72 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/table/a2d427ebc09049d9973c774b71c9af72
2024-11-22T17:38:11,677 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/table/a2d427ebc09049d9973c774b71c9af72, entries=2, sequenceid=17, filesize=5.2 K
2024-11-22T17:38:11,678 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 101ms, sequenceid=17, compaction requested=false
2024-11-22T17:38:11,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T17:38:12,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:12,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36224 deadline: 1732297102823, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. is not online on e61df7af61d2,40173,1732297067182
2024-11-22T17:38:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. is not online on e61df7af61d2,40173,1732297067182
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T17:38:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b. is not online on e61df7af61d2,40173,1732297067182
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-22T17:38:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732297068525.ce559e202d2ab9fd88de1fd31c2bc36b., hostname=e61df7af61d2,40173,1732297067182, seqNum=2 from cache
2024-11-22T17:38:13,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:13,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:14,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:14,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:15,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:15,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:16,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,099 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,100 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,128 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,129 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,134 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,134 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,135 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,138 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:16,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:16,650 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-22T17:38:16,652 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,652 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,653 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,653 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,654 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,654 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,655 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,656 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,679 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,679 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,679 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,686 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-22T17:38:16,985 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-22T17:38:17,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:17,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:18,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:18,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:19,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:19,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:20,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:20,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:21,205 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-22T17:38:21,205 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-22T17:38:21,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:21,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:22,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:22,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:22,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., hostname=e61df7af61d2,40173,1732297067182, seqNum=89]
2024-11-22T17:38:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:22,890 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:22,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7719879a92a148c8a355508f63139db7 is 1080, key is row0065/info:/1732297102877/Put/seqid=0
2024-11-22T17:38:22,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741854_1030 (size=12509)
2024-11-22T17:38:22,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741854_1030 (size=12509)
2024-11-22T17:38:22,902 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7719879a92a148c8a355508f63139db7
2024-11-22T17:38:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7719879a92a148c8a355508f63139db7 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7
2024-11-22T17:38:22,912 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7, entries=7, sequenceid=99, filesize=12.2 K
2024-11-22T17:38:22,913 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=99, compaction requested=false
2024-11-22T17:38:22,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:22,915 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T17:38:22,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/5c3caad73c0b44929359ec60f2000674 is 1080, key is row0072/info:/1732297102892/Put/seqid=0
2024-11-22T17:38:22,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741855_1031 (size=16817)
2024-11-22T17:38:22,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741855_1031 (size=16817)
2024-11-22T17:38:22,925 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/5c3caad73c0b44929359ec60f2000674
2024-11-22T17:38:22,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/5c3caad73c0b44929359ec60f2000674 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674
2024-11-22T17:38:22,937 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674, entries=11, sequenceid=113, filesize=16.4 K
2024-11-22T17:38:22,938 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=113, compaction requested=true
2024-11-22T17:38:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:22,938 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:22,940 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37685 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:22,940 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:22,940 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:22,940 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/8c7c97431359450fabe37aee55ef3c14, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=36.8 K
2024-11-22T17:38:22,940 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T17:38:22,940 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7c97431359450fabe37aee55ef3c14, keycount=3, bloomtype=ROW, size=8.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732297080732
2024-11-22T17:38:22,941 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7719879a92a148c8a355508f63139db7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732297102877
2024-11-22T17:38:22,941 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3caad73c0b44929359ec60f2000674, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732297102892
2024-11-22T17:38:22,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ab8b40c9e3145e7bd20ef5751385f2c is 1080, key is row0083/info:/1732297102916/Put/seqid=0
2024-11-22T17:38:22,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741856_1032 (size=17894)
2024-11-22T17:38:22,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741856_1032 (size=17894)
2024-11-22T17:38:22,949 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ab8b40c9e3145e7bd20ef5751385f2c
2024-11-22T17:38:22,952 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#72 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:22,952 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/ca9708d4fa6440058e95ec1c9efbf232 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:22,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ab8b40c9e3145e7bd20ef5751385f2c as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c
2024-11-22T17:38:22,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741857_1033 (size=27875)
2024-11-22T17:38:22,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741857_1033 (size=27875)
2024-11-22T17:38:22,960 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c, entries=12, sequenceid=128, filesize=17.5 K
2024-11-22T17:38:22,961 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=2.10 KB/2152 for 4b9b5859ba38afad0e83fdf9d5d567af in 21ms, sequenceid=128, compaction requested=false
2024-11-22T17:38:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:22,962 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/ca9708d4fa6440058e95ec1c9efbf232 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ca9708d4fa6440058e95ec1c9efbf232
2024-11-22T17:38:22,968 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into ca9708d4fa6440058e95ec1c9efbf232(size=27.2 K), total size for store is 44.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:22,968 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:22,968 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297102938; duration=0sec
2024-11-22T17:38:22,968 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:22,968 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:23,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:23,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:24,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:24,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:24,964 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:24,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/51026130d4d84d66bfd996d7ee7fe41b is 1080, key is row0095/info:/1732297102942/Put/seqid=0
2024-11-22T17:38:24,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741858_1034 (size=12516)
2024-11-22T17:38:24,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741858_1034 (size=12516)
2024-11-22T17:38:24,977 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/51026130d4d84d66bfd996d7ee7fe41b
2024-11-22T17:38:24,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/51026130d4d84d66bfd996d7ee7fe41b as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b
2024-11-22T17:38:24,990 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b, entries=7, sequenceid=139, filesize=12.2 K
2024-11-22T17:38:24,991 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 4b9b5859ba38afad0e83fdf9d5d567af in 27ms, sequenceid=139, compaction requested=true
2024-11-22T17:38:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:24,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:24,991 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:24,992 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 58285 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:24,992 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:24,992 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:24,992 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T17:38:24,992 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ca9708d4fa6440058e95ec1c9efbf232, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=56.9 K
2024-11-22T17:38:24,993 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca9708d4fa6440058e95ec1c9efbf232, keycount=21, bloomtype=ROW, size=27.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732297080732
2024-11-22T17:38:24,993 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab8b40c9e3145e7bd20ef5751385f2c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732297102916
2024-11-22T17:38:24,993 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51026130d4d84d66bfd996d7ee7fe41b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732297102942
2024-11-22T17:38:24,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7293456d799a4156a64cd6d72c6d8c7f is 1080, key is row0102/info:/1732297104965/Put/seqid=0
2024-11-22T17:38:25,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741859_1035 (size=17906)
2024-11-22T17:38:25,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741859_1035 (size=17906)
2024-11-22T17:38:25,002 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7293456d799a4156a64cd6d72c6d8c7f
2024-11-22T17:38:25,006 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#75 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:25,006 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/09d73f5b837d42daae330e085bc925bf is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:25,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/7293456d799a4156a64cd6d72c6d8c7f as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f
2024-11-22T17:38:25,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741860_1036 (size=48483)
2024-11-22T17:38:25,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741860_1036 (size=48483)
2024-11-22T17:38:25,015 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f, entries=12, sequenceid=154, filesize=17.5 K
2024-11-22T17:38:25,016 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 4b9b5859ba38afad0e83fdf9d5d567af in 24ms, sequenceid=154, compaction requested=false
2024-11-22T17:38:25,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:25,017 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T17:38:25,018 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/09d73f5b837d42daae330e085bc925bf as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/09d73f5b837d42daae330e085bc925bf
2024-11-22T17:38:25,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/aa3bb1e65f3446fa9f117d83677b4201 is 1080, key is row0114/info:/1732297104993/Put/seqid=0
2024-11-22T17:38:25,024 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 09d73f5b837d42daae330e085bc925bf(size=47.3 K), total size for store is 64.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:25,024 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:25,024 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297104991; duration=0sec
2024-11-22T17:38:25,024 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:25,024 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741861_1037 (size=16828)
2024-11-22T17:38:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741861_1037 (size=16828)
2024-11-22T17:38:25,030 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/aa3bb1e65f3446fa9f117d83677b4201
2024-11-22T17:38:25,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/aa3bb1e65f3446fa9f117d83677b4201 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201
2024-11-22T17:38:25,039 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201, entries=11, sequenceid=168, filesize=16.4 K
2024-11-22T17:38:25,040 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=4.20 KB/4304 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=168, compaction requested=true
2024-11-22T17:38:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:25,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:25,040 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:25,041 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 83217 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:25,041 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:25,041 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:25,041 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/09d73f5b837d42daae330e085bc925bf, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=81.3 K
2024-11-22T17:38:25,041 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d73f5b837d42daae330e085bc925bf, keycount=40, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732297080732
2024-11-22T17:38:25,042 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7293456d799a4156a64cd6d72c6d8c7f, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732297104965
2024-11-22T17:38:25,042 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa3bb1e65f3446fa9f117d83677b4201, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732297104993
2024-11-22T17:38:25,051 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#77 average throughput is 32.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:25,051 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/078014139b4f4c09960e6c720a295f86 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:25,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741862_1038 (size=73520)
2024-11-22T17:38:25,055 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741862_1038 (size=73520)
2024-11-22T17:38:25,060 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/078014139b4f4c09960e6c720a295f86 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/078014139b4f4c09960e6c720a295f86
2024-11-22T17:38:25,066 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 078014139b4f4c09960e6c720a295f86(size=71.8 K), total size for store is 71.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:25,066 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:25,066 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297105040; duration=0sec
2024-11-22T17:38:25,066 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:25,066 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:25,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:25,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:26,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:26,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:27,038 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:27,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/dce56bc4a6ec418caee80fb2cdef60e5 is 1080, key is row0125/info:/1732297105019/Put/seqid=0
2024-11-22T17:38:27,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741863_1039 (size=12516)
2024-11-22T17:38:27,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741863_1039 (size=12516)
2024-11-22T17:38:27,049 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/dce56bc4a6ec418caee80fb2cdef60e5
2024-11-22T17:38:27,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/dce56bc4a6ec418caee80fb2cdef60e5 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5
2024-11-22T17:38:27,061 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5, entries=7, sequenceid=180, filesize=12.2 K
2024-11-22T17:38:27,062 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 4b9b5859ba38afad0e83fdf9d5d567af in 24ms, sequenceid=180, compaction requested=false
2024-11-22T17:38:27,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:27,064 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T17:38:27,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/6f22e5618d094863b10b9d2a00326613 is 1080, key is row0132/info:/1732297107039/Put/seqid=0
2024-11-22T17:38:27,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741864_1040 (size=19000)
2024-11-22T17:38:27,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741864_1040 (size=19000)
2024-11-22T17:38:27,074 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/6f22e5618d094863b10b9d2a00326613
2024-11-22T17:38:27,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/6f22e5618d094863b10b9d2a00326613 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613
2024-11-22T17:38:27,086 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613, entries=13, sequenceid=196, filesize=18.6 K
2024-11-22T17:38:27,087 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=196, compaction requested=true
2024-11-22T17:38:27,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:27,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:27,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:27,087 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:27,088 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-22T17:38:27,088 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 105036 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:27,089 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:27,089 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:27,089 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/078014139b4f4c09960e6c720a295f86, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=102.6 K
2024-11-22T17:38:27,089 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078014139b4f4c09960e6c720a295f86, keycount=63, bloomtype=ROW, size=71.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732297080732
2024-11-22T17:38:27,090 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce56bc4a6ec418caee80fb2cdef60e5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732297105019
2024-11-22T17:38:27,090 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f22e5618d094863b10b9d2a00326613, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732297107039
2024-11-22T17:38:27,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/644a263e15354339ab56ac3c848dffb6 is 1080, key is row0145/info:/1732297107065/Put/seqid=0
2024-11-22T17:38:27,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741865_1041 (size=17906)
2024-11-22T17:38:27,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741865_1041 (size=17906)
2024-11-22T17:38:27,101 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/644a263e15354339ab56ac3c848dffb6
2024-11-22T17:38:27,106 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#81 average throughput is 28.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:27,106 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9309541ddd6146f1bac475bc2b020675 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:27,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/644a263e15354339ab56ac3c848dffb6 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6
2024-11-22T17:38:27,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741866_1042 (size=95186)
2024-11-22T17:38:27,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741866_1042 (size=95186)
2024-11-22T17:38:27,114 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6, entries=12, sequenceid=211, filesize=17.5 K
2024-11-22T17:38:27,115 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=4.20 KB/4304 for 4b9b5859ba38afad0e83fdf9d5d567af in 27ms, sequenceid=211, compaction requested=false
2024-11-22T17:38:27,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:27,117 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9309541ddd6146f1bac475bc2b020675 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9309541ddd6146f1bac475bc2b020675
2024-11-22T17:38:27,123 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 9309541ddd6146f1bac475bc2b020675(size=93.0 K), total size for store is 110.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:27,123 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:27,123 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297107087; duration=0sec
2024-11-22T17:38:27,123 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:27,123 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:27,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:27,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:28,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:28,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:28,966 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-22T17:38:28,966 INFO  [master/e61df7af61d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-22T17:38:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:29,112 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:29,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/fd06aae5e2fd48d39e5fb95a0fa5a32c is 1080, key is row0157/info:/1732297107090/Put/seqid=0
2024-11-22T17:38:29,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741867_1043 (size=12516)
2024-11-22T17:38:29,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741867_1043 (size=12516)
2024-11-22T17:38:29,123 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/fd06aae5e2fd48d39e5fb95a0fa5a32c
2024-11-22T17:38:29,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/fd06aae5e2fd48d39e5fb95a0fa5a32c as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c
2024-11-22T17:38:29,134 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c, entries=7, sequenceid=222, filesize=12.2 K
2024-11-22T17:38:29,135 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=222, compaction requested=true
2024-11-22T17:38:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:29,135 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:29,136 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 125608 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:29,136 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:29,136 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:29,136 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9309541ddd6146f1bac475bc2b020675, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=122.7 K
2024-11-22T17:38:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:29,137 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9309541ddd6146f1bac475bc2b020675, keycount=83, bloomtype=ROW, size=93.0 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732297080732
2024-11-22T17:38:29,137 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T17:38:29,137 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644a263e15354339ab56ac3c848dffb6, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732297107065
2024-11-22T17:38:29,138 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd06aae5e2fd48d39e5fb95a0fa5a32c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732297107090
2024-11-22T17:38:29,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2000f0ff8c9347e39950f92d5a03eed3 is 1080, key is row0164/info:/1732297109114/Put/seqid=0
2024-11-22T17:38:29,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741868_1044 (size=16828)
2024-11-22T17:38:29,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741868_1044 (size=16828)
2024-11-22T17:38:29,150 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2000f0ff8c9347e39950f92d5a03eed3
2024-11-22T17:38:29,152 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#84 average throughput is 34.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:29,153 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/ac7c2ba24fa346d4b632e18d7f5a59a3 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2000f0ff8c9347e39950f92d5a03eed3 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3
2024-11-22T17:38:29,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741869_1045 (size=115774)
2024-11-22T17:38:29,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741869_1045 (size=115774)
2024-11-22T17:38:29,164 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3, entries=11, sequenceid=236, filesize=16.4 K
2024-11-22T17:38:29,164 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for 4b9b5859ba38afad0e83fdf9d5d567af in 27ms, sequenceid=236, compaction requested=false
2024-11-22T17:38:29,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:29,166 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-22T17:38:29,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/806551cf078648da9c56b687c55ef33b is 1080, key is row0175/info:/1732297109138/Put/seqid=0
2024-11-22T17:38:29,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741870_1046 (size=20078)
2024-11-22T17:38:29,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741870_1046 (size=20078)
2024-11-22T17:38:29,175 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/806551cf078648da9c56b687c55ef33b
2024-11-22T17:38:29,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/806551cf078648da9c56b687c55ef33b as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b
2024-11-22T17:38:29,184 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b, entries=14, sequenceid=253, filesize=19.6 K
2024-11-22T17:38:29,185 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=4.20 KB/4304 for 4b9b5859ba38afad0e83fdf9d5d567af in 19ms, sequenceid=253, compaction requested=false
2024-11-22T17:38:29,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:29,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:29,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:29,630 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/ac7c2ba24fa346d4b632e18d7f5a59a3 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ac7c2ba24fa346d4b632e18d7f5a59a3
2024-11-22T17:38:29,638 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into ac7c2ba24fa346d4b632e18d7f5a59a3(size=113.1 K), total size for store is 149.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:29,638 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:29,639 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297109135; duration=0sec
2024-11-22T17:38:29,639 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:29,639 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:30,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:30,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:31,186 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:31,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/26ee5ea350c549f484b1b4a990286b37 is 1080, key is row0189/info:/1732297109167/Put/seqid=0
2024-11-22T17:38:31,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741871_1047 (size=12522)
2024-11-22T17:38:31,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741871_1047 (size=12522)
2024-11-22T17:38:31,196 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/26ee5ea350c549f484b1b4a990286b37
2024-11-22T17:38:31,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/26ee5ea350c549f484b1b4a990286b37 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37
2024-11-22T17:38:31,208 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37, entries=7, sequenceid=264, filesize=12.2 K
2024-11-22T17:38:31,209 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 4b9b5859ba38afad0e83fdf9d5d567af in 23ms, sequenceid=264, compaction requested=true
2024-11-22T17:38:31,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:31,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:31,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:31,210 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-22T17:38:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:31,211 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T17:38:31,211 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4  files of size 165202 starting at candidate #0 after considering 3 permutations with 3 in ratio
2024-11-22T17:38:31,211 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:31,211 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:31,211 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ac7c2ba24fa346d4b632e18d7f5a59a3, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=161.3 K
2024-11-22T17:38:31,212 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac7c2ba24fa346d4b632e18d7f5a59a3, keycount=102, bloomtype=ROW, size=113.1 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732297080732
2024-11-22T17:38:31,212 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2000f0ff8c9347e39950f92d5a03eed3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732297109114
2024-11-22T17:38:31,213 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806551cf078648da9c56b687c55ef33b, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732297109138
2024-11-22T17:38:31,213 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ee5ea350c549f484b1b4a990286b37, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732297109167
2024-11-22T17:38:31,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/571562b3a9224a98bc6b2ca3ce9d9316 is 1080, key is row0196/info:/1732297111187/Put/seqid=0
2024-11-22T17:38:31,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741872_1048 (size=19013)
2024-11-22T17:38:31,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741872_1048 (size=19013)
2024-11-22T17:38:31,220 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/571562b3a9224a98bc6b2ca3ce9d9316
2024-11-22T17:38:31,227 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#88 average throughput is 45.83 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:31,227 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/a1f5c36e319844f2a2173f41d5481f59 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:31,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/571562b3a9224a98bc6b2ca3ce9d9316 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316
2024-11-22T17:38:31,233 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316, entries=13, sequenceid=280, filesize=18.6 K
2024-11-22T17:38:31,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741873_1049 (size=150617)
2024-11-22T17:38:31,235 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=12.61 KB/12912 for 4b9b5859ba38afad0e83fdf9d5d567af in 24ms, sequenceid=280, compaction requested=false
2024-11-22T17:38:31,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:31,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741873_1049 (size=150617)
2024-11-22T17:38:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:31,239 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T17:38:31,240 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/a1f5c36e319844f2a2173f41d5481f59 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a1f5c36e319844f2a2173f41d5481f59
2024-11-22T17:38:31,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/cff4dec4670b49a19a9a44a6551e2746 is 1080, key is row0209/info:/1732297111212/Put/seqid=0
2024-11-22T17:38:31,247 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into a1f5c36e319844f2a2173f41d5481f59(size=147.1 K), total size for store is 165.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:31,247 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:31,247 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=12, startTime=1732297111209; duration=0sec
2024-11-22T17:38:31,247 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:31,247 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:31,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741874_1050 (size=19013)
2024-11-22T17:38:31,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741874_1050 (size=19013)
2024-11-22T17:38:31,250 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/cff4dec4670b49a19a9a44a6551e2746
2024-11-22T17:38:31,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/cff4dec4670b49a19a9a44a6551e2746 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746
2024-11-22T17:38:31,258 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746, entries=13, sequenceid=296, filesize=18.6 K
2024-11-22T17:38:31,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=3.15 KB/3228 for 4b9b5859ba38afad0e83fdf9d5d567af in 20ms, sequenceid=296, compaction requested=true
2024-11-22T17:38:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:31,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:31,259 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:31,261 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 188643 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:31,261 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:31,261 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:31,261 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a1f5c36e319844f2a2173f41d5481f59, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=184.2 K
2024-11-22T17:38:31,261 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f5c36e319844f2a2173f41d5481f59, keycount=134, bloomtype=ROW, size=147.1 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732297080732
2024-11-22T17:38:31,262 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 571562b3a9224a98bc6b2ca3ce9d9316, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732297111187
2024-11-22T17:38:31,262 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting cff4dec4670b49a19a9a44a6551e2746, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732297111212
2024-11-22T17:38:31,274 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#90 average throughput is 54.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:31,274 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/761cf46413dc4146992f2ade08e7b830 is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:31,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741875_1051 (size=178781)
2024-11-22T17:38:31,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741875_1051 (size=178781)
2024-11-22T17:38:31,282 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/761cf46413dc4146992f2ade08e7b830 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/761cf46413dc4146992f2ade08e7b830
2024-11-22T17:38:31,288 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 761cf46413dc4146992f2ade08e7b830(size=174.6 K), total size for store is 174.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:31,288 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:31,288 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297111259; duration=0sec
2024-11-22T17:38:31,288 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:31,288 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:31,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:31,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:32,597 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:32,597 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:33,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-22T17:38:33,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ac4c7c04cd64006b6d476ffc6fadfe1 is 1080, key is row0222/info:/1732297111241/Put/seqid=0
2024-11-22T17:38:33,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741876_1052 (size=12523)
2024-11-22T17:38:33,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741876_1052 (size=12523)
2024-11-22T17:38:33,273 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ac4c7c04cd64006b6d476ffc6fadfe1
2024-11-22T17:38:33,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/2ac4c7c04cd64006b6d476ffc6fadfe1 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1
2024-11-22T17:38:33,283 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1, entries=7, sequenceid=308, filesize=12.2 K
2024-11-22T17:38:33,284 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 4b9b5859ba38afad0e83fdf9d5d567af in 24ms, sequenceid=308, compaction requested=false
2024-11-22T17:38:33,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:33,284 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-22T17:38:33,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/835ef7e5462e4cceb6e92ab040387199 is 1080, key is row0229/info:/1732297113260/Put/seqid=0
2024-11-22T17:38:33,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741877_1053 (size=19013)
2024-11-22T17:38:33,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741877_1053 (size=19013)
2024-11-22T17:38:33,294 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/835ef7e5462e4cceb6e92ab040387199
2024-11-22T17:38:33,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/835ef7e5462e4cceb6e92ab040387199 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199
2024-11-22T17:38:33,304 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199, entries=13, sequenceid=324, filesize=18.6 K
2024-11-22T17:38:33,305 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for 4b9b5859ba38afad0e83fdf9d5d567af in 21ms, sequenceid=324, compaction requested=true
2024-11-22T17:38:33,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:33,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9b5859ba38afad0e83fdf9d5d567af:info, priority=-2147483648, current under compaction store size is 1
2024-11-22T17:38:33,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:33,305 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-22T17:38:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] regionserver.HRegion(8855): Flush requested on 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:33,305 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-22T17:38:33,306 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 210317 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-22T17:38:33,306 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1541): 4b9b5859ba38afad0e83fdf9d5d567af/info is initiating minor compaction (all files)
2024-11-22T17:38:33,306 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9b5859ba38afad0e83fdf9d5d567af/info in TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:33,306 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/761cf46413dc4146992f2ade08e7b830, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199] into tmpdir=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp, totalSize=205.4 K
2024-11-22T17:38:33,307 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761cf46413dc4146992f2ade08e7b830, keycount=160, bloomtype=ROW, size=174.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732297080732
2024-11-22T17:38:33,307 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ac4c7c04cd64006b6d476ffc6fadfe1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732297111241
2024-11-22T17:38:33,308 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835ef7e5462e4cceb6e92ab040387199, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732297113260
2024-11-22T17:38:33,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/3a5844f96b3143d99d9e9e6d591cf1df is 1080, key is row0242/info:/1732297113286/Put/seqid=0
2024-11-22T17:38:33,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741878_1054 (size=16839)
2024-11-22T17:38:33,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741878_1054 (size=16839)
2024-11-22T17:38:33,314 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/3a5844f96b3143d99d9e9e6d591cf1df
2024-11-22T17:38:33,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/3a5844f96b3143d99d9e9e6d591cf1df as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/3a5844f96b3143d99d9e9e6d591cf1df
2024-11-22T17:38:33,322 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9b5859ba38afad0e83fdf9d5d567af#info#compaction#94 average throughput is 46.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-22T17:38:33,322 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9d81eb92407a4f54a568130e00695f7a is 1080, key is row0062/info:/1732297080732/Put/seqid=0
2024-11-22T17:38:33,323 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/3a5844f96b3143d99d9e9e6d591cf1df, entries=11, sequenceid=338, filesize=16.4 K
2024-11-22T17:38:33,324 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=4.20 KB/4304 for 4b9b5859ba38afad0e83fdf9d5d567af in 19ms, sequenceid=338, compaction requested=false
2024-11-22T17:38:33,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:33,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741879_1055 (size=200467)
2024-11-22T17:38:33,325 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741879_1055 (size=200467)
2024-11-22T17:38:33,331 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9d81eb92407a4f54a568130e00695f7a as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9d81eb92407a4f54a568130e00695f7a
2024-11-22T17:38:33,336 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9b5859ba38afad0e83fdf9d5d567af/info of 4b9b5859ba38afad0e83fdf9d5d567af into 9d81eb92407a4f54a568130e00695f7a(size=195.8 K), total size for store is 212.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-22T17:38:33,336 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:33,336 INFO  [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., storeName=4b9b5859ba38afad0e83fdf9d5d567af/info, priority=13, startTime=1732297113305; duration=0sec
2024-11-22T17:38:33,336 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-22T17:38:33,336 DEBUG [RS:0;e61df7af61d2:40173-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9b5859ba38afad0e83fdf9d5d567af:info
2024-11-22T17:38:33,404 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-11-22T17:38:33,598 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:33,598 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:34,599 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:34,599 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:35,314 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-22T17:38:35,315 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C40173%2C1732297067182.1732297115314
2024-11-22T17:38:35,326 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,326 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,326 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,326 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,327 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,327 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297067811 with entries=320, filesize=310.93 KB; new WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297115314
2024-11-22T17:38:35,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33497:33497),(127.0.0.1/127.0.0.1:40237:40237)]
2024-11-22T17:38:35,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297067811 is not closed yet, will try archiving it next time
2024-11-22T17:38:35,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741833_1009 (size=318403)
2024-11-22T17:38:35,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741833_1009 (size=318403)
2024-11-22T17:38:35,332 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a8048cdbcee3d4f4fb7b457f3ee6e4db:

2024-11-22T17:38:35,332 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b9b5859ba38afad0e83fdf9d5d567af 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-22T17:38:35,336 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9716545e32bd4140908f408a3481533a is 1080, key is row0253/info:/1732297113306/Put/seqid=0
2024-11-22T17:38:35,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741881_1057 (size=9278)
2024-11-22T17:38:35,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741881_1057 (size=9278)
2024-11-22T17:38:35,341 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9716545e32bd4140908f408a3481533a
2024-11-22T17:38:35,346 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/.tmp/info/9716545e32bd4140908f408a3481533a as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9716545e32bd4140908f408a3481533a
2024-11-22T17:38:35,351 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9716545e32bd4140908f408a3481533a, entries=4, sequenceid=346, filesize=9.1 K
2024-11-22T17:38:35,352 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 4b9b5859ba38afad0e83fdf9d5d567af in 20ms, sequenceid=346, compaction requested=true
2024-11-22T17:38:35,352 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4b9b5859ba38afad0e83fdf9d5d567af:

2024-11-22T17:38:35,352 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-22T17:38:35,356 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/bb166fe524f343b0b31aaba2e9038ac9 is 193, key is TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af./info:regioninfo/1732297091599/Put/seqid=0
2024-11-22T17:38:35,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741882_1058 (size=6223)
2024-11-22T17:38:35,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741882_1058 (size=6223)
2024-11-22T17:38:35,360 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/bb166fe524f343b0b31aaba2e9038ac9
2024-11-22T17:38:35,365 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/.tmp/info/bb166fe524f343b0b31aaba2e9038ac9 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/info/bb166fe524f343b0b31aaba2e9038ac9
2024-11-22T17:38:35,370 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/info/bb166fe524f343b0b31aaba2e9038ac9, entries=5, sequenceid=21, filesize=6.1 K
2024-11-22T17:38:35,371 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-11-22T17:38:35,371 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-22T17:38:35,371 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C40173%2C1732297067182.1732297115371
2024-11-22T17:38:35,375 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,375 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,376 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,376 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,376 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:35,376 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297115314 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297115371
2024-11-22T17:38:35,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40237:40237),(127.0.0.1/127.0.0.1:33497:33497)]
2024-11-22T17:38:35,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297115314 is not closed yet, will try archiving it next time
2024-11-22T17:38:35,377 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297067811 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs/e61df7af61d2%2C40173%2C1732297067182.1732297067811
2024-11-22T17:38:35,378 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-22T17:38:35,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741880_1056 (size=731)
2024-11-22T17:38:35,378 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741880_1056 (size=731)
2024-11-22T17:38:35,600 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:35,600 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:35,780 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/WALs/e61df7af61d2,40173,1732297067182/e61df7af61d2%2C40173%2C1732297067182.1732297115314 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs/e61df7af61d2%2C40173%2C1732297067182.1732297115314
2024-11-22T17:38:35,879 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:38:35,879 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:38:35,879 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:35,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:35,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:35,879 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:38:35,879 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:38:35,879 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1096471280, stopped=false
2024-11-22T17:38:35,880 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,33625,1732297067006
2024-11-22T17:38:35,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:35,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:35,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:35,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:35,951 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:38:35,952 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:38:35,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:35,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:35,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:35,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:35,954 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,40173,1732297067182' *****
2024-11-22T17:38:35,954 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:38:35,955 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(3091): Received CLOSE for a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b9b5859ba38afad0e83fdf9d5d567af
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,40173,1732297067182
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:38:35,955 INFO  [RS:0;e61df7af61d2:40173 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:40173.
2024-11-22T17:38:35,955 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a8048cdbcee3d4f4fb7b457f3ee6e4db, disabling compactions & flushes
2024-11-22T17:38:35,955 DEBUG [RS:0;e61df7af61d2:40173 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:35,955 DEBUG [RS:0;e61df7af61d2:40173 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:35,955 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:35,956 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:35,956 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:38:35,956 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:38:35,956 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db. after waiting 0 ms
2024-11-22T17:38:35,956 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:38:35,956 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:35,956 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:38:35,956 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-22T17:38:35,956 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1325): Online Regions={a8048cdbcee3d4f4fb7b457f3ee6e4db=TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db., 4b9b5859ba38afad0e83fdf9d5d567af=TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af., 1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:38:35,957 DEBUG [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4b9b5859ba38afad0e83fdf9d5d567af, a8048cdbcee3d4f4fb7b457f3ee6e4db
2024-11-22T17:38:35,957 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:38:35,957 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:38:35,957 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:38:35,957 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:38:35,956 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-bottom] to archive
2024-11-22T17:38:35,957 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:38:35,958 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:38:35,961 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:35,961 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e61df7af61d2:33625 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-22T17:38:35,961 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-22T17:38:35,963 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-22T17:38:35,963 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:38:35,963 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:38:35,963 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297115957Running coprocessor pre-close hooks at 1732297115957Disabling compacts and flushes for region at 1732297115957Disabling writes for close at 1732297115957Writing region close event to WAL at 1732297115959 (+2 ms)Running coprocessor post-close hooks at 1732297115963 (+4 ms)Closed at 1732297115963
2024-11-22T17:38:35,963 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:38:35,965 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/a8048cdbcee3d4f4fb7b457f3ee6e4db/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=88
2024-11-22T17:38:35,965 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:35,965 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a8048cdbcee3d4f4fb7b457f3ee6e4db:
Waiting for close lock at 1732297115955Running coprocessor pre-close hooks at 1732297115955Disabling compacts and flushes for region at 1732297115955Disabling writes for close at 1732297115956 (+1 ms)Writing region close event to WAL at 1732297115962 (+6 ms)Running coprocessor post-close hooks at 1732297115965 (+3 ms)Closed at 1732297115965
2024-11-22T17:38:35,965 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732297090883.a8048cdbcee3d4f4fb7b457f3ee6e4db.
2024-11-22T17:38:35,966 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b9b5859ba38afad0e83fdf9d5d567af, disabling compactions & flushes
2024-11-22T17:38:35,966 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:35,966 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:35,966 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af. after waiting 0 ms
2024-11-22T17:38:35,966 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:35,966 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b->hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/ce559e202d2ab9fd88de1fd31c2bc36b/info/a48a4f595e014b0f94db7ac4f3e8cec5-top, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/8c7c97431359450fabe37aee55ef3c14, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ca9708d4fa6440058e95ec1c9efbf232, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/09d73f5b837d42daae330e085bc925bf, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/078014139b4f4c09960e6c720a295f86, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9309541ddd6146f1bac475bc2b020675, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ac7c2ba24fa346d4b632e18d7f5a59a3, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a1f5c36e319844f2a2173f41d5481f59, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/761cf46413dc4146992f2ade08e7b830, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199] to archive
2024-11-22T17:38:35,967 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-22T17:38:35,968 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a48a4f595e014b0f94db7ac4f3e8cec5.ce559e202d2ab9fd88de1fd31c2bc36b
2024-11-22T17:38:35,969 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/8c7c97431359450fabe37aee55ef3c14 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/8c7c97431359450fabe37aee55ef3c14
2024-11-22T17:38:35,970 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/TestLogRolling-testLogRolling=ce559e202d2ab9fd88de1fd31c2bc36b-a27c043181fc4db3901f54a0e54e2e1e
2024-11-22T17:38:35,971 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7719879a92a148c8a355508f63139db7
2024-11-22T17:38:35,972 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ca9708d4fa6440058e95ec1c9efbf232 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ca9708d4fa6440058e95ec1c9efbf232
2024-11-22T17:38:35,973 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/5c3caad73c0b44929359ec60f2000674
2024-11-22T17:38:35,974 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ab8b40c9e3145e7bd20ef5751385f2c
2024-11-22T17:38:35,975 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/09d73f5b837d42daae330e085bc925bf to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/09d73f5b837d42daae330e085bc925bf
2024-11-22T17:38:35,976 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/51026130d4d84d66bfd996d7ee7fe41b
2024-11-22T17:38:35,977 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/7293456d799a4156a64cd6d72c6d8c7f
2024-11-22T17:38:35,978 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/078014139b4f4c09960e6c720a295f86 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/078014139b4f4c09960e6c720a295f86
2024-11-22T17:38:35,979 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/aa3bb1e65f3446fa9f117d83677b4201
2024-11-22T17:38:35,980 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/dce56bc4a6ec418caee80fb2cdef60e5
2024-11-22T17:38:35,981 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9309541ddd6146f1bac475bc2b020675 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/9309541ddd6146f1bac475bc2b020675
2024-11-22T17:38:35,982 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/6f22e5618d094863b10b9d2a00326613
2024-11-22T17:38:35,983 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/644a263e15354339ab56ac3c848dffb6
2024-11-22T17:38:35,984 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ac7c2ba24fa346d4b632e18d7f5a59a3 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/ac7c2ba24fa346d4b632e18d7f5a59a3
2024-11-22T17:38:35,984 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/fd06aae5e2fd48d39e5fb95a0fa5a32c
2024-11-22T17:38:35,985 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2000f0ff8c9347e39950f92d5a03eed3
2024-11-22T17:38:35,986 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/806551cf078648da9c56b687c55ef33b
2024-11-22T17:38:35,987 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a1f5c36e319844f2a2173f41d5481f59 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/a1f5c36e319844f2a2173f41d5481f59
2024-11-22T17:38:35,988 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/26ee5ea350c549f484b1b4a990286b37
2024-11-22T17:38:35,989 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/571562b3a9224a98bc6b2ca3ce9d9316
2024-11-22T17:38:35,990 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/761cf46413dc4146992f2ade08e7b830 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/761cf46413dc4146992f2ade08e7b830
2024-11-22T17:38:35,990 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/cff4dec4670b49a19a9a44a6551e2746
2024-11-22T17:38:35,991 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/2ac4c7c04cd64006b6d476ffc6fadfe1
2024-11-22T17:38:35,992 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199 to hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/archive/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/info/835ef7e5462e4cceb6e92ab040387199
2024-11-22T17:38:35,992 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8c7c97431359450fabe37aee55ef3c14=8359, 7719879a92a148c8a355508f63139db7=12509, ca9708d4fa6440058e95ec1c9efbf232=27875, 5c3caad73c0b44929359ec60f2000674=16817, 2ab8b40c9e3145e7bd20ef5751385f2c=17894, 09d73f5b837d42daae330e085bc925bf=48483, 51026130d4d84d66bfd996d7ee7fe41b=12516, 7293456d799a4156a64cd6d72c6d8c7f=17906, 078014139b4f4c09960e6c720a295f86=73520, aa3bb1e65f3446fa9f117d83677b4201=16828, dce56bc4a6ec418caee80fb2cdef60e5=12516, 9309541ddd6146f1bac475bc2b020675=95186, 6f22e5618d094863b10b9d2a00326613=19000, 644a263e15354339ab56ac3c848dffb6=17906, ac7c2ba24fa346d4b632e18d7f5a59a3=115774, fd06aae5e2fd48d39e5fb95a0fa5a32c=12516, 2000f0ff8c9347e39950f92d5a03eed3=16828, 806551cf078648da9c56b687c55ef33b=20078, a1f5c36e319844f2a2173f41d5481f59=150617, 26ee5ea350c549f484b1b4a990286b37=12522, 571562b3a9224a98bc6b2ca3ce9d9316=19013, 761cf46413dc4146992f2ade08e7b830=178781, cff4dec4670b49a19a9a44a6551e2746=19013, 2ac4c7c04cd64006b6d476ffc6fadfe1=12523, 835ef7e5462e4cceb6e92ab040387199=19013]
2024-11-22T17:38:35,996 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/data/default/TestLogRolling-testLogRolling/4b9b5859ba38afad0e83fdf9d5d567af/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=88
2024-11-22T17:38:35,997 INFO  [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:35,997 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b9b5859ba38afad0e83fdf9d5d567af:
Waiting for close lock at 1732297115966Running coprocessor pre-close hooks at 1732297115966Disabling compacts and flushes for region at 1732297115966Disabling writes for close at 1732297115966Writing region close event to WAL at 1732297115993 (+27 ms)Running coprocessor post-close hooks at 1732297115996 (+3 ms)Closed at 1732297115996
2024-11-22T17:38:35,997 DEBUG [RS_CLOSE_REGION-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732297090883.4b9b5859ba38afad0e83fdf9d5d567af.
2024-11-22T17:38:36,157 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,40173,1732297067182; all regions closed.
2024-11-22T17:38:36,157 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,158 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,158 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,158 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,158 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741834_1010 (size=8107)
2024-11-22T17:38:36,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741834_1010 (size=8107)
2024-11-22T17:38:36,162 DEBUG [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs
2024-11-22T17:38:36,162 INFO  [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C40173%2C1732297067182.meta:.meta(num 1732297068394)
2024-11-22T17:38:36,162 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,162 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,162 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,163 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,163 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741883_1059 (size=778)
2024-11-22T17:38:36,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741883_1059 (size=778)
2024-11-22T17:38:36,166 DEBUG [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/oldWALs
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C40173%2C1732297067182:(num 1732297115371)
2024-11-22T17:38:36,166 DEBUG [RS:0;e61df7af61d2:40173 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:38:36,166 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:38:36,166 INFO  [RS:0;e61df7af61d2:40173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40173
2024-11-22T17:38:36,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,40173,1732297067182
2024-11-22T17:38:36,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:38:36,173 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:38:36,184 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,40173,1732297067182]
2024-11-22T17:38:36,195 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,40173,1732297067182 already deleted, retry=false
2024-11-22T17:38:36,195 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,40173,1732297067182 expired; onlineServers=0
2024-11-22T17:38:36,195 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,33625,1732297067006' *****
2024-11-22T17:38:36,195 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:38:36,195 INFO  [M:0;e61df7af61d2:33625 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:38:36,195 INFO  [M:0;e61df7af61d2:33625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:38:36,195 DEBUG [M:0;e61df7af61d2:33625 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:38:36,195 DEBUG [M:0;e61df7af61d2:33625 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:38:36,195 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:38:36,195 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297067553 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297067553,5,FailOnTimeoutGroup]
2024-11-22T17:38:36,195 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297067553 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297067553,5,FailOnTimeoutGroup]
2024-11-22T17:38:36,196 INFO  [M:0;e61df7af61d2:33625 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:38:36,196 INFO  [M:0;e61df7af61d2:33625 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:38:36,196 DEBUG [M:0;e61df7af61d2:33625 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:38:36,196 INFO  [M:0;e61df7af61d2:33625 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:38:36,196 INFO  [M:0;e61df7af61d2:33625 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:38:36,196 INFO  [M:0;e61df7af61d2:33625 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:38:36,196 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:38:36,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:38:36,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:36,205 DEBUG [M:0;e61df7af61d2:33625 {}] zookeeper.ZKUtil(347): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:38:36,205 WARN  [M:0;e61df7af61d2:33625 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:38:36,206 INFO  [M:0;e61df7af61d2:33625 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/.lastflushedseqids
2024-11-22T17:38:36,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741884_1060 (size=228)
2024-11-22T17:38:36,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741884_1060 (size=228)
2024-11-22T17:38:36,216 INFO  [M:0;e61df7af61d2:33625 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:38:36,216 INFO  [M:0;e61df7af61d2:33625 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:38:36,216 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:38:36,216 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:36,216 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:36,217 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:38:36,217 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:36,217 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.36 KB
2024-11-22T17:38:36,235 DEBUG [M:0;e61df7af61d2:33625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1b243e693d8457ab2c45ee5c7a54537 is 82, key is hbase:meta,,1/info:regioninfo/1732297068420/Put/seqid=0
2024-11-22T17:38:36,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741885_1061 (size=5672)
2024-11-22T17:38:36,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741885_1061 (size=5672)
2024-11-22T17:38:36,239 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1b243e693d8457ab2c45ee5c7a54537
2024-11-22T17:38:36,259 DEBUG [M:0;e61df7af61d2:33625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3a76ffd709f42de8c9b2a674f4e002e is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732297068904/Put/seqid=0
2024-11-22T17:38:36,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741886_1062 (size=7091)
2024-11-22T17:38:36,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741886_1062 (size=7091)
2024-11-22T17:38:36,263 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3a76ffd709f42de8c9b2a674f4e002e
2024-11-22T17:38:36,267 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c3a76ffd709f42de8c9b2a674f4e002e
2024-11-22T17:38:36,283 DEBUG [M:0;e61df7af61d2:33625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58897a05263b456287fac3115f80a512 is 69, key is e61df7af61d2,40173,1732297067182/rs:state/1732297067653/Put/seqid=0
2024-11-22T17:38:36,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:36,284 INFO  [RS:0;e61df7af61d2:40173 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:38:36,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40173-0x101639f85980001, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:36,284 INFO  [RS:0;e61df7af61d2:40173 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,40173,1732297067182; zookeeper connection closed.
2024-11-22T17:38:36,285 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@530e3c6b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@530e3c6b
2024-11-22T17:38:36,285 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:38:36,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741887_1063 (size=5156)
2024-11-22T17:38:36,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741887_1063 (size=5156)
2024-11-22T17:38:36,288 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58897a05263b456287fac3115f80a512
2024-11-22T17:38:36,305 DEBUG [M:0;e61df7af61d2:33625 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f340f24fc34607aee73c744ea4b31e is 52, key is load_balancer_on/state:d/1732297068521/Put/seqid=0
2024-11-22T17:38:36,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741888_1064 (size=5056)
2024-11-22T17:38:36,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741888_1064 (size=5056)
2024-11-22T17:38:36,309 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f340f24fc34607aee73c744ea4b31e
2024-11-22T17:38:36,313 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1b243e693d8457ab2c45ee5c7a54537 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1b243e693d8457ab2c45ee5c7a54537
2024-11-22T17:38:36,318 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1b243e693d8457ab2c45ee5c7a54537, entries=8, sequenceid=125, filesize=5.5 K
2024-11-22T17:38:36,319 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c3a76ffd709f42de8c9b2a674f4e002e as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c3a76ffd709f42de8c9b2a674f4e002e
2024-11-22T17:38:36,323 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c3a76ffd709f42de8c9b2a674f4e002e
2024-11-22T17:38:36,323 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c3a76ffd709f42de8c9b2a674f4e002e, entries=13, sequenceid=125, filesize=6.9 K
2024-11-22T17:38:36,324 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58897a05263b456287fac3115f80a512 as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58897a05263b456287fac3115f80a512
2024-11-22T17:38:36,329 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58897a05263b456287fac3115f80a512, entries=1, sequenceid=125, filesize=5.0 K
2024-11-22T17:38:36,330 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f340f24fc34607aee73c744ea4b31e as hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/71f340f24fc34607aee73c744ea4b31e
2024-11-22T17:38:36,334 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42755/user/jenkins/test-data/8172c9e6-8b71-6b3e-ed89-02bf19eb5e59/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/71f340f24fc34607aee73c744ea4b31e, entries=1, sequenceid=125, filesize=4.9 K
2024-11-22T17:38:36,335 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=125, compaction requested=false
2024-11-22T17:38:36,336 INFO  [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:36,336 DEBUG [M:0;e61df7af61d2:33625 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297116216Disabling compacts and flushes for region at 1732297116216Disabling writes for close at 1732297116217 (+1 ms)Obtaining lock to block concurrent updates at 1732297116217Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732297116217Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1732297116217Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732297116218 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732297116218Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732297116235 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732297116235Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732297116243 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732297116259 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732297116259Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732297116267 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732297116282 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732297116282Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732297116291 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732297116304 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732297116304Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77215df1: reopening flushed file at 1732297116312 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32fc3aa2: reopening flushed file at 1732297116318 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fd1b9e8: reopening flushed file at 1732297116324 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5279d9ab: reopening flushed file at 1732297116329 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=125, compaction requested=false at 1732297116335 (+6 ms)Writing region close event to WAL at 1732297116336 (+1 ms)Closed at 1732297116336
2024-11-22T17:38:36,337 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,337 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,337 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,337 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,337 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:36,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36371 is added to blk_1073741830_1006 (size=61332)
2024-11-22T17:38:36,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44599 is added to blk_1073741830_1006 (size=61332)
2024-11-22T17:38:36,339 INFO  [M:0;e61df7af61d2:33625 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:38:36,339 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:38:36,339 INFO  [M:0;e61df7af61d2:33625 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33625
2024-11-22T17:38:36,339 INFO  [M:0;e61df7af61d2:33625 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:38:36,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:36,451 INFO  [M:0;e61df7af61d2:33625 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:38:36,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33625-0x101639f85980000, quorum=127.0.0.1:62481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:36,457 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4cfa1960{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:36,457 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7289fa1e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:36,457 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:36,458 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dbdcf8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:36,458 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51d47a1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:36,460 WARN  [BP-413928561-172.17.0.2-1732297064796 heartbeating to localhost/127.0.0.1:42755 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:38:36,460 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:38:36,460 WARN  [BP-413928561-172.17.0.2-1732297064796 heartbeating to localhost/127.0.0.1:42755 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-413928561-172.17.0.2-1732297064796 (Datanode Uuid d9274d56-39bc-4353-b62d-b734295cb6a3) service to localhost/127.0.0.1:42755
2024-11-22T17:38:36,460 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:38:36,461 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data3/current/BP-413928561-172.17.0.2-1732297064796 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:36,461 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data4/current/BP-413928561-172.17.0.2-1732297064796 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:36,461 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:38:36,464 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@278c53c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:36,464 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a127189{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:36,464 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:36,464 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a5fe763{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:36,464 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef16bcf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:36,466 WARN  [BP-413928561-172.17.0.2-1732297064796 heartbeating to localhost/127.0.0.1:42755 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:38:36,466 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:38:36,466 WARN  [BP-413928561-172.17.0.2-1732297064796 heartbeating to localhost/127.0.0.1:42755 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-413928561-172.17.0.2-1732297064796 (Datanode Uuid 837ef63f-5c78-4b25-be8f-5aa961fe1140) service to localhost/127.0.0.1:42755
2024-11-22T17:38:36,466 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:38:36,467 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data1/current/BP-413928561-172.17.0.2-1732297064796 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:36,467 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/cluster_808b1ef5-9f9d-3e03-a442-c7700b3f31d2/data/data2/current/BP-413928561-172.17.0.2-1732297064796 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:36,467 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:38:36,473 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5350cad{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:38:36,473 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@278f505{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:36,473 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:36,473 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ec6f4ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:36,473 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6150e164{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:36,480 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:38:36,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:38:36,533 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 205)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42755
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:42755 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:42755 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42755
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42755
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42755
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:42755 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:42755
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=524 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=179 (was 188), ProcessCount=11 (was 11), AvailableMemoryMB=2417 (was 1641) - AvailableMemoryMB LEAK? -
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=524, MaxFileDescriptor=1048576, SystemLoadAverage=179, ProcessCount=11, AvailableMemoryMB=2417
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.log.dir so I do NOT create it in target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2e35c859-f2be-a247-79fa-328148e093d1/hadoop.tmp.dir so I do NOT create it in target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce, deleteOnExit=true
2024-11-22T17:38:36,540 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/test.cache.data in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.tmp.dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-22T17:38:36,541 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:38:36,541 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/nfs.dump.dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/java.io.tmpdir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-22T17:38:36,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-22T17:38:36,554 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:38:36,600 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:36,600 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:36,939 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:38:36,943 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:38:36,945 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:38:36,945 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:38:36,945 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:38:36,946 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:38:36,946 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24e1a3ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:38:36,946 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a903086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:38:37,038 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@293ad0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/java.io.tmpdir/jetty-localhost-42397-hadoop-hdfs-3_4_1-tests_jar-_-any-4289299326483770153/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:38:37,038 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@72a3013{HTTP/1.1, (http/1.1)}{localhost:42397}
2024-11-22T17:38:37,038 INFO  [Time-limited test {}] server.Server(415): Started @303076ms
2024-11-22T17:38:37,048 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-22T17:38:37,300 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:38:37,304 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:38:37,305 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:38:37,305 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:38:37,305 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:38:37,305 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@676ee024{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:38:37,305 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7df47ec9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:38:37,397 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64cb4406{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/java.io.tmpdir/jetty-localhost-36979-hadoop-hdfs-3_4_1-tests_jar-_-any-4043101567447311527/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:37,397 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27b8ac4b{HTTP/1.1, (http/1.1)}{localhost:36979}
2024-11-22T17:38:37,397 INFO  [Time-limited test {}] server.Server(415): Started @303435ms
2024-11-22T17:38:37,398 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:38:37,421 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-22T17:38:37,424 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-22T17:38:37,425 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-22T17:38:37,425 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-22T17:38:37,425 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-22T17:38:37,425 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13f7f7ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,AVAILABLE}
2024-11-22T17:38:37,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf154bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-22T17:38:37,517 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@610f8a79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/java.io.tmpdir/jetty-localhost-36957-hadoop-hdfs-3_4_1-tests_jar-_-any-18321939446520356813/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:37,517 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50ea3f07{HTTP/1.1, (http/1.1)}{localhost:36957}
2024-11-22T17:38:37,517 INFO  [Time-limited test {}] server.Server(415): Started @303555ms
2024-11-22T17:38:37,518 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-22T17:38:37,601 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:37,601 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:37,675 INFO  [regionserver/e61df7af61d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:38:38,558 WARN  [Thread-2528 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data1/current/BP-519871691-172.17.0.2-1732297116558/current, will proceed with Du for space computation calculation, 
2024-11-22T17:38:38,558 WARN  [Thread-2529 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data2/current/BP-519871691-172.17.0.2-1732297116558/current, will proceed with Du for space computation calculation, 
2024-11-22T17:38:38,578 WARN  [Thread-2492 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:38:38,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61009b6dd7d2eddc with lease ID 0xd48c200c08433f6b: Processing first storage report for DS-47decbe2-b7dc-4ee5-aeea-016d27c6c746 from datanode DatanodeRegistration(127.0.0.1:45299, datanodeUuid=c17b0a50-c86f-4cce-8d83-de443f6c42b9, infoPort=41411, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558)
2024-11-22T17:38:38,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61009b6dd7d2eddc with lease ID 0xd48c200c08433f6b: from storage DS-47decbe2-b7dc-4ee5-aeea-016d27c6c746 node DatanodeRegistration(127.0.0.1:45299, datanodeUuid=c17b0a50-c86f-4cce-8d83-de443f6c42b9, infoPort=41411, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:38:38,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61009b6dd7d2eddc with lease ID 0xd48c200c08433f6b: Processing first storage report for DS-0142c09c-1758-4d64-a2de-4e20aa3294ed from datanode DatanodeRegistration(127.0.0.1:45299, datanodeUuid=c17b0a50-c86f-4cce-8d83-de443f6c42b9, infoPort=41411, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558)
2024-11-22T17:38:38,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61009b6dd7d2eddc with lease ID 0xd48c200c08433f6b: from storage DS-0142c09c-1758-4d64-a2de-4e20aa3294ed node DatanodeRegistration(127.0.0.1:45299, datanodeUuid=c17b0a50-c86f-4cce-8d83-de443f6c42b9, infoPort=41411, infoSecurePort=0, ipcPort=40715, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:38:38,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:38,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:38,693 WARN  [Thread-2539 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data3/current/BP-519871691-172.17.0.2-1732297116558/current, will proceed with Du for space computation calculation, 
2024-11-22T17:38:38,693 WARN  [Thread-2540 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data4/current/BP-519871691-172.17.0.2-1732297116558/current, will proceed with Du for space computation calculation, 
2024-11-22T17:38:38,712 WARN  [Thread-2515 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-22T17:38:38,714 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb760e0938ee28c3b with lease ID 0xd48c200c08433f6c: Processing first storage report for DS-2bacb4f9-1411-4959-9ed8-3a96487d91ad from datanode DatanodeRegistration(127.0.0.1:35993, datanodeUuid=9ff0beb1-7ec8-4096-893a-dacf92478b00, infoPort=39693, infoSecurePort=0, ipcPort=37591, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558)
2024-11-22T17:38:38,714 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb760e0938ee28c3b with lease ID 0xd48c200c08433f6c: from storage DS-2bacb4f9-1411-4959-9ed8-3a96487d91ad node DatanodeRegistration(127.0.0.1:35993, datanodeUuid=9ff0beb1-7ec8-4096-893a-dacf92478b00, infoPort=39693, infoSecurePort=0, ipcPort=37591, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:38:38,714 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb760e0938ee28c3b with lease ID 0xd48c200c08433f6c: Processing first storage report for DS-102aae97-55da-407b-a350-ca7508ca4b6b from datanode DatanodeRegistration(127.0.0.1:35993, datanodeUuid=9ff0beb1-7ec8-4096-893a-dacf92478b00, infoPort=39693, infoSecurePort=0, ipcPort=37591, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558)
2024-11-22T17:38:38,714 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb760e0938ee28c3b with lease ID 0xd48c200c08433f6c: from storage DS-102aae97-55da-407b-a350-ca7508ca4b6b node DatanodeRegistration(127.0.0.1:35993, datanodeUuid=9ff0beb1-7ec8-4096-893a-dacf92478b00, infoPort=39693, infoSecurePort=0, ipcPort=37591, storageInfo=lv=-57;cid=testClusterID;nsid=1835362950;c=1732297116558), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-22T17:38:38,748 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88
2024-11-22T17:38:38,752 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/zookeeper_0, clientPort=56137, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-22T17:38:38,754 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56137
2024-11-22T17:38:38,754 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,755 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:38:38,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741825_1001 (size=7)
2024-11-22T17:38:38,769 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f with version=8
2024-11-22T17:38:38,769 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:41389/user/jenkins/test-data/f80f85c4-7378-bcc3-4653-ee0e1ada255b/hbase-staging
2024-11-22T17:38:38,771 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-22T17:38:38,772 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:38:38,773 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33881
2024-11-22T17:38:38,774 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33881 connecting to ZooKeeper ensemble=127.0.0.1:56137
2024-11-22T17:38:38,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:338810x0, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:38:38,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33881-0x10163a04fc90000 connected
2024-11-22T17:38:38,900 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,901 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,903 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:38,903 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f, hbase.cluster.distributed=false
2024-11-22T17:38:38,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:38:38,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33881
2024-11-22T17:38:38,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33881
2024-11-22T17:38:38,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33881
2024-11-22T17:38:38,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33881
2024-11-22T17:38:38,907 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33881
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e61df7af61d2:0 server-side Connection retries=45
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-22T17:38:38,924 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-22T17:38:38,925 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39375
2024-11-22T17:38:38,926 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39375 connecting to ZooKeeper ensemble=127.0.0.1:56137
2024-11-22T17:38:38,926 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,927 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:38,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393750x0, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-22T17:38:38,942 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39375-0x10163a04fc90001 connected
2024-11-22T17:38:38,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:38,942 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-22T17:38:38,943 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-22T17:38:38,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-22T17:38:38,945 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-22T17:38:38,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39375
2024-11-22T17:38:38,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39375
2024-11-22T17:38:38,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39375
2024-11-22T17:38:38,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39375
2024-11-22T17:38:38,946 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39375
2024-11-22T17:38:38,960 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e61df7af61d2:33881
2024-11-22T17:38:38,960 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:38:39,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:38:39,040 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-22T17:38:39,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,106 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-22T17:38:39,108 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e61df7af61d2,33881,1732297118771 from backup master directory
2024-11-22T17:38:39,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:38:39,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-22T17:38:39,132 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:38:39,132 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,151 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/hbase.id] with ID: 1d1539c5-dd23-43f1-b34a-90ef26bd246c
2024-11-22T17:38:39,151 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/.tmp/hbase.id
2024-11-22T17:38:39,155 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:38:39,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741826_1002 (size=42)
2024-11-22T17:38:39,156 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/.tmp/hbase.id]:[hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/hbase.id]
2024-11-22T17:38:39,168 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:39,168 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-22T17:38:39,170 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-22T17:38:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:38:39,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741827_1003 (size=196)
2024-11-22T17:38:39,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-22T17:38:39,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-22T17:38:39,190 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:38:39,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:38:39,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741828_1004 (size=1189)
2024-11-22T17:38:39,200 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store
2024-11-22T17:38:39,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:38:39,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741829_1005 (size=34)
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:38:39,207 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:39,207 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:39,207 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297119207Disabling compacts and flushes for region at 1732297119207Disabling writes for close at 1732297119207Writing region close event to WAL at 1732297119207Closed at 1732297119207
2024-11-22T17:38:39,208 WARN  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/.initializing
2024-11-22T17:38:39,208 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/WALs/e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,210 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C33881%2C1732297118771, suffix=, logDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/WALs/e61df7af61d2,33881,1732297118771, archiveDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/oldWALs, maxLogs=10
2024-11-22T17:38:39,211 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C33881%2C1732297118771.1732297119210
2024-11-22T17:38:39,217 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/WALs/e61df7af61d2,33881,1732297118771/e61df7af61d2%2C33881%2C1732297118771.1732297119210
2024-11-22T17:38:39,218 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39693:39693),(127.0.0.1/127.0.0.1:41411:41411)]
2024-11-22T17:38:39,219 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:38:39,219 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:39,219 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,219 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,221 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:38:39,222 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-22T17:38:39,222 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,223 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:39,223 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:38:39,224 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-22T17:38:39,224 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,225 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:38:39,225 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:38:39,226 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-22T17:38:39,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,226 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:38:39,226 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-22T17:38:39,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-22T17:38:39,227 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,227 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-22T17:38:39,227 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,228 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,228 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,229 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,229 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,229 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-22T17:38:39,230 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-22T17:38:39,232 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:38:39,232 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=858129, jitterRate=0.09116844832897186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-22T17:38:39,232 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732297119219Initializing all the Stores at 1732297119220 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297119220Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297119220Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297119220Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297119220Cleaning up temporary data from old regions at 1732297119229 (+9 ms)Region opened successfully at 1732297119232 (+3 ms)
2024-11-22T17:38:39,233 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-22T17:38:39,235 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a8dbd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:38:39,236 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-22T17:38:39,236 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-22T17:38:39,236 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-22T17:38:39,236 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-22T17:38:39,236 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-22T17:38:39,237 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-22T17:38:39,237 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-22T17:38:39,239 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-22T17:38:39,239 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-22T17:38:39,247 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-22T17:38:39,247 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-22T17:38:39,248 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-22T17:38:39,257 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-22T17:38:39,258 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-22T17:38:39,259 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-22T17:38:39,268 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-22T17:38:39,269 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-22T17:38:39,278 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-22T17:38:39,281 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-22T17:38:39,289 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-22T17:38:39,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:39,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:39,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,300 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e61df7af61d2,33881,1732297118771, sessionid=0x10163a04fc90000, setting cluster-up flag (Was=false)
2024-11-22T17:38:39,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,352 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-22T17:38:39,354 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,405 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-22T17:38:39,409 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e61df7af61d2,33881,1732297118771
2024-11-22T17:38:39,412 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-22T17:38:39,415 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-22T17:38:39,415 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-22T17:38:39,415 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-22T17:38:39,415 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e61df7af61d2,33881,1732297118771
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-22T17:38:39,417 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e61df7af61d2:0, corePoolSize=5, maxPoolSize=5
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e61df7af61d2:0, corePoolSize=10, maxPoolSize=10
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:38:39,418 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732297149419
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-22T17:38:39,419 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,420 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:38:39,420 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-22T17:38:39,420 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-22T17:38:39,420 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-22T17:38:39,420 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-22T17:38:39,420 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-22T17:38:39,420 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-22T17:38:39,421 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297119420,5,FailOnTimeoutGroup]
2024-11-22T17:38:39,421 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297119421,5,FailOnTimeoutGroup]
2024-11-22T17:38:39,421 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,421 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,421 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-22T17:38:39,421 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,421 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,421 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-22T17:38:39,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:38:39,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741831_1007 (size=1321)
2024-11-22T17:38:39,428 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-22T17:38:39,428 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f
2024-11-22T17:38:39,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:38:39,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741832_1008 (size=32)
2024-11-22T17:38:39,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:39,440 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:38:39,441 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:38:39,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,441 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:39,441 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:38:39,442 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:38:39,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,442 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:39,442 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:38:39,443 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:38:39,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,443 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:39,444 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:38:39,444 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:38:39,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:39,445 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:39,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:38:39,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740
2024-11-22T17:38:39,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740
2024-11-22T17:38:39,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:38:39,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:38:39,447 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:38:39,448 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(746): ClusterId : 1d1539c5-dd23-43f1-b34a-90ef26bd246c
2024-11-22T17:38:39,448 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-22T17:38:39,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:38:39,450 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-22T17:38:39,450 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690598, jitterRate=-0.12186001241207123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732297119439Initializing all the Stores at 1732297119439Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297119439Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297119440 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297119440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297119440Cleaning up temporary data from old regions at 1732297119447 (+7 ms)Region opened successfully at 1732297119451 (+4 ms)
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:38:39,451 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:38:39,451 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:38:39,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297119451Disabling compacts and flushes for region at 1732297119451Disabling writes for close at 1732297119451Writing region close event to WAL at 1732297119451Closed at 1732297119451
2024-11-22T17:38:39,452 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:38:39,452 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-22T17:38:39,452 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-22T17:38:39,453 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:38:39,454 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-22T17:38:39,516 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-22T17:38:39,516 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-22T17:38:39,590 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-22T17:38:39,591 DEBUG [RS:0;e61df7af61d2:39375 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fd2bdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e61df7af61d2/172.17.0.2:0
2024-11-22T17:38:39,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:39,602 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:39,604 WARN  [e61df7af61d2:33881 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-22T17:38:39,608 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e61df7af61d2:39375
2024-11-22T17:38:39,608 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-22T17:38:39,608 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-22T17:38:39,608 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-22T17:38:39,609 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(2659): reportForDuty to master=e61df7af61d2,33881,1732297118771 with port=39375, startcode=1732297118924
2024-11-22T17:38:39,609 DEBUG [RS:0;e61df7af61d2:39375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-22T17:38:39,611 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40015, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-22T17:38:39,611 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33881 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,612 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33881 {}] master.ServerManager(517): Registering regionserver=e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,613 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f
2024-11-22T17:38:39,613 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46159
2024-11-22T17:38:39,613 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-22T17:38:39,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:38:39,637 DEBUG [RS:0;e61df7af61d2:39375 {}] zookeeper.ZKUtil(111): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,637 WARN  [RS:0;e61df7af61d2:39375 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-22T17:38:39,637 INFO  [RS:0;e61df7af61d2:39375 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:38:39,637 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,638 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e61df7af61d2,39375,1732297118924]
2024-11-22T17:38:39,642 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-22T17:38:39,644 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-22T17:38:39,645 INFO  [RS:0;e61df7af61d2:39375 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-22T17:38:39,645 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,645 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-22T17:38:39,646 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-22T17:38:39,646 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e61df7af61d2:0, corePoolSize=2, maxPoolSize=2
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,647 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,648 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e61df7af61d2:0, corePoolSize=1, maxPoolSize=1
2024-11-22T17:38:39,648 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:38:39,648 DEBUG [RS:0;e61df7af61d2:39375 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e61df7af61d2:0, corePoolSize=3, maxPoolSize=3
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,648 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,39375,1732297118924-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:38:39,661 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-22T17:38:39,661 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,39375,1732297118924-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,661 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,661 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.Replication(171): e61df7af61d2,39375,1732297118924 started
2024-11-22T17:38:39,673 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:39,673 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1482): Serving as e61df7af61d2,39375,1732297118924, RpcServer on e61df7af61d2/172.17.0.2:39375, sessionid=0x10163a04fc90001
2024-11-22T17:38:39,674 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-22T17:38:39,674 DEBUG [RS:0;e61df7af61d2:39375 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,674 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,39375,1732297118924'
2024-11-22T17:38:39,674 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-22T17:38:39,674 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e61df7af61d2,39375,1732297118924'
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-22T17:38:39,675 DEBUG [RS:0;e61df7af61d2:39375 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-22T17:38:39,675 INFO  [RS:0;e61df7af61d2:39375 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-22T17:38:39,675 INFO  [RS:0;e61df7af61d2:39375 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-22T17:38:39,779 INFO  [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C39375%2C1732297118924, suffix=, logDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/e61df7af61d2,39375,1732297118924, archiveDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs, maxLogs=32
2024-11-22T17:38:39,780 INFO  [RS:0;e61df7af61d2:39375 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C39375%2C1732297118924.1732297119779
2024-11-22T17:38:39,789 INFO  [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/e61df7af61d2,39375,1732297118924/e61df7af61d2%2C39375%2C1732297118924.1732297119779
2024-11-22T17:38:39,790 DEBUG [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41411:41411),(127.0.0.1/127.0.0.1:39693:39693)]
2024-11-22T17:38:39,854 DEBUG [e61df7af61d2:33881 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-22T17:38:39,855 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e61df7af61d2,39375,1732297118924
2024-11-22T17:38:39,858 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,39375,1732297118924, state=OPENING
2024-11-22T17:38:39,868 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-22T17:38:39,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:39,881 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-22T17:38:39,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:38:39,881 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:38:39,881 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,39375,1732297118924}]
2024-11-22T17:38:40,036 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-22T17:38:40,038 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33245, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-22T17:38:40,042 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-22T17:38:40,042 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:38:40,043 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e61df7af61d2%2C39375%2C1732297118924.meta, suffix=.meta, logDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/e61df7af61d2,39375,1732297118924, archiveDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs, maxLogs=32
2024-11-22T17:38:40,044 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e61df7af61d2%2C39375%2C1732297118924.meta.1732297120044.meta
2024-11-22T17:38:40,048 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/e61df7af61d2,39375,1732297118924/e61df7af61d2%2C39375%2C1732297118924.meta.1732297120044.meta
2024-11-22T17:38:40,050 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41411:41411),(127.0.0.1/127.0.0.1:39693:39693)]
2024-11-22T17:38:40,052 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-22T17:38:40,053 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-22T17:38:40,053 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-22T17:38:40,054 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-22T17:38:40,055 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-22T17:38:40,055 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:40,056 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:40,056 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-22T17:38:40,056 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-22T17:38:40,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:40,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:40,057 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-22T17:38:40,057 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-22T17:38:40,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:40,058 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:40,058 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-22T17:38:40,058 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-22T17:38:40,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-22T17:38:40,059 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-22T17:38:40,059 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-22T17:38:40,060 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740
2024-11-22T17:38:40,060 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740
2024-11-22T17:38:40,062 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-22T17:38:40,062 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-22T17:38:40,062 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-22T17:38:40,064 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-22T17:38:40,064 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859252, jitterRate=0.0925956666469574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-22T17:38:40,064 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-22T17:38:40,065 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732297120053Writing region info on filesystem at 1732297120053Initializing all the Stores at 1732297120054 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297120054Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297120054Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732297120054Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732297120054Cleaning up temporary data from old regions at 1732297120062 (+8 ms)Running coprocessor post-open hooks at 1732297120064 (+2 ms)Region opened successfully at 1732297120065 (+1 ms)
2024-11-22T17:38:40,066 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732297120036
2024-11-22T17:38:40,068 DEBUG [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-22T17:38:40,068 INFO  [RS_OPEN_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-22T17:38:40,068 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e61df7af61d2,39375,1732297118924
2024-11-22T17:38:40,069 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e61df7af61d2,39375,1732297118924, state=OPEN
2024-11-22T17:38:40,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:38:40,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-22T17:38:40,163 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e61df7af61d2,39375,1732297118924
2024-11-22T17:38:40,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:38:40,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-22T17:38:40,168 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-22T17:38:40,168 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e61df7af61d2,39375,1732297118924 in 282 msec
2024-11-22T17:38:40,173 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-22T17:38:40,173 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 716 msec
2024-11-22T17:38:40,175 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-22T17:38:40,175 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-22T17:38:40,177 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:38:40,177 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,39375,1732297118924, seqNum=-1]
2024-11-22T17:38:40,178 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:38:40,179 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58061, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:38:40,186 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 771 msec
2024-11-22T17:38:40,186 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732297120186, completionTime=-1
2024-11-22T17:38:40,186 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-22T17:38:40,186 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-22T17:38:40,188 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-22T17:38:40,188 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732297180188
2024-11-22T17:38:40,188 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732297240188
2024-11-22T17:38:40,188 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e61df7af61d2:33881, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,189 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,191 DEBUG [master/e61df7af61d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.061sec
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-22T17:38:40,193 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-22T17:38:40,196 DEBUG [master/e61df7af61d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-22T17:38:40,196 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-22T17:38:40,196 INFO  [master/e61df7af61d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e61df7af61d2,33881,1732297118771-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-22T17:38:40,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1414536c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:38:40,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e61df7af61d2,33881,-1 for getting cluster id
2024-11-22T17:38:40,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-22T17:38:40,251 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1d1539c5-dd23-43f1-b34a-90ef26bd246c'
2024-11-22T17:38:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-22T17:38:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1d1539c5-dd23-43f1-b34a-90ef26bd246c"

2024-11-22T17:38:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c35653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:38:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e61df7af61d2,33881,-1]
2024-11-22T17:38:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-22T17:38:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,253 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-22T17:38:40,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bdd10f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-22T17:38:40,255 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-22T17:38:40,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e61df7af61d2,39375,1732297118924, seqNum=-1]
2024-11-22T17:38:40,256 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-22T17:38:40,257 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-22T17:38:40,259 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e61df7af61d2,33881,1732297118771
2024-11-22T17:38:40,260 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-22T17:38:40,263 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-22T17:38:40,263 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-22T17:38:40,266 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1, archiveDir=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs, maxLogs=32
2024-11-22T17:38:40,267 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732297120267
2024-11-22T17:38:40,273 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732297120267
2024-11-22T17:38:40,274 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41411:41411),(127.0.0.1/127.0.0.1:39693:39693)]
2024-11-22T17:38:40,275 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732297120275
2024-11-22T17:38:40,281 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,281 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,281 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,281 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,281 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,281 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732297120267 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732297120275
2024-11-22T17:38:40,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41411:41411),(127.0.0.1/127.0.0.1:39693:39693)]
2024-11-22T17:38:40,283 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732297120267 is not closed yet, will try archiving it next time
2024-11-22T17:38:40,283 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741835_1011 (size=93)
2024-11-22T17:38:40,283 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,284 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,284 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,284 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741835_1011 (size=93)
2024-11-22T17:38:40,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741836_1012 (size=93)
2024-11-22T17:38:40,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741836_1012 (size=93)
2024-11-22T17:38:40,288 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732297120267 to hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs/test.com%2C8080%2C1.1732297120267
2024-11-22T17:38:40,290 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs
2024-11-22T17:38:40,290 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732297120275)
2024-11-22T17:38:40,290 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-22T17:38:40,290 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:38:40,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:40,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,291 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-22T17:38:40,291 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-22T17:38:40,291 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=717065623, stopped=false
2024-11-22T17:38:40,291 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e61df7af61d2,33881,1732297118771
2024-11-22T17:38:40,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:40,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-22T17:38:40,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:40,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:40,310 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:38:40,310 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-22T17:38:40,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:40,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,311 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e61df7af61d2,39375,1732297118924' *****
2024-11-22T17:38:40,311 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-22T17:38:40,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:40,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-22T17:38:40,311 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-22T17:38:40,311 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-22T17:38:40,311 INFO  [RS:0;e61df7af61d2:39375 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(959): stopping server e61df7af61d2,39375,1732297118924
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e61df7af61d2:39375.
2024-11-22T17:38:40,312 DEBUG [RS:0;e61df7af61d2:39375 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-22T17:38:40,312 DEBUG [RS:0;e61df7af61d2:39375 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-22T17:38:40,312 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-22T17:38:40,313 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-22T17:38:40,313 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-22T17:38:40,313 DEBUG [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-22T17:38:40,313 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-22T17:38:40,313 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-22T17:38:40,313 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-22T17:38:40,313 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-22T17:38:40,313 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-22T17:38:40,313 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-22T17:38:40,332 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/.tmp/ns/10edde42a3494805a88954bdf3cefd09 is 43, key is default/ns:d/1732297120180/Put/seqid=0
2024-11-22T17:38:40,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741837_1013 (size=5153)
2024-11-22T17:38:40,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741837_1013 (size=5153)
2024-11-22T17:38:40,337 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/.tmp/ns/10edde42a3494805a88954bdf3cefd09
2024-11-22T17:38:40,342 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/.tmp/ns/10edde42a3494805a88954bdf3cefd09 as hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/ns/10edde42a3494805a88954bdf3cefd09
2024-11-22T17:38:40,346 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/ns/10edde42a3494805a88954bdf3cefd09, entries=2, sequenceid=6, filesize=5.0 K
2024-11-22T17:38:40,347 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false
2024-11-22T17:38:40,350 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-22T17:38:40,351 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-22T17:38:40,351 INFO  [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-22T17:38:40,351 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732297120313Running coprocessor pre-close hooks at 1732297120313Disabling compacts and flushes for region at 1732297120313Disabling writes for close at 1732297120313Obtaining lock to block concurrent updates at 1732297120313Preparing flush snapshotting stores in 1588230740 at 1732297120313Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732297120314 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732297120314Flushing 1588230740/ns: creating writer at 1732297120315 (+1 ms)Flushing 1588230740/ns: appending metadata at 1732297120331 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732297120331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c33031a: reopening flushed file at 1732297120341 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 34ms, sequenceid=6, compaction requested=false at 1732297120347 (+6 ms)Writing region close event to WAL at 1732297120347Running coprocessor post-close hooks at 1732297120351 (+4 ms)Closed at 1732297120351
2024-11-22T17:38:40,351 DEBUG [RS_CLOSE_META-regionserver/e61df7af61d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-22T17:38:40,513 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(976): stopping server e61df7af61d2,39375,1732297118924; all regions closed.
2024-11-22T17:38:40,513 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,514 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,514 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,514 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,514 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,515 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741834_1010 (size=1152)
2024-11-22T17:38:40,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741834_1010 (size=1152)
2024-11-22T17:38:40,517 DEBUG [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs
2024-11-22T17:38:40,517 INFO  [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C39375%2C1732297118924.meta:.meta(num 1732297120044)
2024-11-22T17:38:40,518 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,518 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,518 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,518 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,518 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741833_1009 (size=93)
2024-11-22T17:38:40,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741833_1009 (size=93)
2024-11-22T17:38:40,521 DEBUG [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/oldWALs
2024-11-22T17:38:40,521 INFO  [RS:0;e61df7af61d2:39375 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e61df7af61d2%2C39375%2C1732297118924:(num 1732297119779)
2024-11-22T17:38:40,521 DEBUG [RS:0;e61df7af61d2:39375 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-22T17:38:40,521 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.LeaseManager(133): Closed leases
2024-11-22T17:38:40,522 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:38:40,522 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.ChoreService(370): Chore service for: regionserver/e61df7af61d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-22T17:38:40,522 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:38:40,522 INFO  [regionserver/e61df7af61d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:38:40,522 INFO  [RS:0;e61df7af61d2:39375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39375
2024-11-22T17:38:40,531 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:38:40,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-22T17:38:40,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e61df7af61d2,39375,1732297118924
2024-11-22T17:38:40,541 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e61df7af61d2,39375,1732297118924]
2024-11-22T17:38:40,552 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e61df7af61d2,39375,1732297118924 already deleted, retry=false
2024-11-22T17:38:40,552 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e61df7af61d2,39375,1732297118924 expired; onlineServers=0
2024-11-22T17:38:40,552 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e61df7af61d2,33881,1732297118771' *****
2024-11-22T17:38:40,552 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-22T17:38:40,552 INFO  [M:0;e61df7af61d2:33881 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-22T17:38:40,552 INFO  [M:0;e61df7af61d2:33881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-22T17:38:40,553 DEBUG [M:0;e61df7af61d2:33881 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-22T17:38:40,553 DEBUG [M:0;e61df7af61d2:33881 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-22T17:38:40,553 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-22T17:38:40,553 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297119421 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.small.0-1732297119421,5,FailOnTimeoutGroup]
2024-11-22T17:38:40,553 DEBUG [master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297119420 {}] cleaner.HFileCleaner(306): Exit Thread[master/e61df7af61d2:0:becomeActiveMaster-HFileCleaner.large.0-1732297119420,5,FailOnTimeoutGroup]
2024-11-22T17:38:40,553 INFO  [M:0;e61df7af61d2:33881 {}] hbase.ChoreService(370): Chore service for: master/e61df7af61d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-22T17:38:40,553 INFO  [M:0;e61df7af61d2:33881 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-22T17:38:40,553 DEBUG [M:0;e61df7af61d2:33881 {}] master.HMaster(1795): Stopping service threads
2024-11-22T17:38:40,553 INFO  [M:0;e61df7af61d2:33881 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-22T17:38:40,553 INFO  [M:0;e61df7af61d2:33881 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-22T17:38:40,553 INFO  [M:0;e61df7af61d2:33881 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-22T17:38:40,554 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-22T17:38:40,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-22T17:38:40,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-22T17:38:40,563 DEBUG [M:0;e61df7af61d2:33881 {}] zookeeper.ZKUtil(347): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-22T17:38:40,563 WARN  [M:0;e61df7af61d2:33881 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-22T17:38:40,564 INFO  [M:0;e61df7af61d2:33881 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/.lastflushedseqids
2024-11-22T17:38:40,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741838_1014 (size=99)
2024-11-22T17:38:40,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741838_1014 (size=99)
2024-11-22T17:38:40,572 INFO  [M:0;e61df7af61d2:33881 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-22T17:38:40,572 INFO  [M:0;e61df7af61d2:33881 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-22T17:38:40,572 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-22T17:38:40,573 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:40,573 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:40,573 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-22T17:38:40,573 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:40,573 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-22T17:38:40,591 DEBUG [M:0;e61df7af61d2:33881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc58ebe5b3f48548ef1ea08407a3e42 is 82, key is hbase:meta,,1/info:regioninfo/1732297120068/Put/seqid=0
2024-11-22T17:38:40,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741839_1015 (size=5672)
2024-11-22T17:38:40,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741839_1015 (size=5672)
2024-11-22T17:38:40,596 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc58ebe5b3f48548ef1ea08407a3e42
2024-11-22T17:38:40,603 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,41147,1732296927067/e61df7af61d2%2C41147%2C1732296927067.1732296927294
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:40,603 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:33455/user/jenkins/test-data/637b9457-9fbb-77e2-065f-59a6020d0c24/WALs/e61df7af61d2,34069,1732296925813/e61df7af61d2%2C34069%2C1732296925813.meta.1732296926879.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-22T17:38:40,613 DEBUG [M:0;e61df7af61d2:33881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2057c3750a3c4e79a28828358370c99b is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732297120185/Put/seqid=0
2024-11-22T17:38:40,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741840_1016 (size=5275)
2024-11-22T17:38:40,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741840_1016 (size=5275)
2024-11-22T17:38:40,618 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2057c3750a3c4e79a28828358370c99b
2024-11-22T17:38:40,636 DEBUG [M:0;e61df7af61d2:33881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a93a13340004c3ebfac181602fca269 is 69, key is e61df7af61d2,39375,1732297118924/rs:state/1732297119612/Put/seqid=0
2024-11-22T17:38:40,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741841_1017 (size=5156)
2024-11-22T17:38:40,640 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741841_1017 (size=5156)
2024-11-22T17:38:40,641 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a93a13340004c3ebfac181602fca269
2024-11-22T17:38:40,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:40,642 INFO  [RS:0;e61df7af61d2:39375 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:38:40,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39375-0x10163a04fc90001, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:40,642 INFO  [RS:0;e61df7af61d2:39375 {}] regionserver.HRegionServer(1031): Exiting; stopping=e61df7af61d2,39375,1732297118924; zookeeper connection closed.
2024-11-22T17:38:40,642 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6adb543e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6adb543e
2024-11-22T17:38:40,642 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-22T17:38:40,657 DEBUG [M:0;e61df7af61d2:33881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d90c7acc82a8422183a78285dfad7ce5 is 52, key is load_balancer_on/state:d/1732297120262/Put/seqid=0
2024-11-22T17:38:40,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741842_1018 (size=5056)
2024-11-22T17:38:40,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741842_1018 (size=5056)
2024-11-22T17:38:40,662 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d90c7acc82a8422183a78285dfad7ce5
2024-11-22T17:38:40,666 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/adc58ebe5b3f48548ef1ea08407a3e42 as hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/adc58ebe5b3f48548ef1ea08407a3e42
2024-11-22T17:38:40,669 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/adc58ebe5b3f48548ef1ea08407a3e42, entries=8, sequenceid=29, filesize=5.5 K
2024-11-22T17:38:40,670 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2057c3750a3c4e79a28828358370c99b as hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2057c3750a3c4e79a28828358370c99b
2024-11-22T17:38:40,674 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2057c3750a3c4e79a28828358370c99b, entries=3, sequenceid=29, filesize=5.2 K
2024-11-22T17:38:40,675 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6a93a13340004c3ebfac181602fca269 as hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a93a13340004c3ebfac181602fca269
2024-11-22T17:38:40,680 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6a93a13340004c3ebfac181602fca269, entries=1, sequenceid=29, filesize=5.0 K
2024-11-22T17:38:40,681 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d90c7acc82a8422183a78285dfad7ce5 as hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d90c7acc82a8422183a78285dfad7ce5
2024-11-22T17:38:40,685 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46159/user/jenkins/test-data/d57acb83-c50c-1092-4a3d-a0ca9f6aa87f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d90c7acc82a8422183a78285dfad7ce5, entries=1, sequenceid=29, filesize=4.9 K
2024-11-22T17:38:40,686 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=29, compaction requested=false
2024-11-22T17:38:40,687 INFO  [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-22T17:38:40,688 DEBUG [M:0;e61df7af61d2:33881 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732297120572Disabling compacts and flushes for region at 1732297120572Disabling writes for close at 1732297120573 (+1 ms)Obtaining lock to block concurrent updates at 1732297120573Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732297120573Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732297120573Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732297120574 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732297120574Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732297120591 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732297120591Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732297120600 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732297120613 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732297120613Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732297120621 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732297120635 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732297120635Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732297120645 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732297120657 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732297120657Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@677cf99e: reopening flushed file at 1732297120665 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4878fcc8: reopening flushed file at 1732297120670 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b7ca32c: reopening flushed file at 1732297120675 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571501b7: reopening flushed file at 1732297120680 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=29, compaction requested=false at 1732297120686 (+6 ms)Writing region close event to WAL at 1732297120687 (+1 ms)Closed at 1732297120687
2024-11-22T17:38:40,688 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,688 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,688 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,688 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,688 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-22T17:38:40,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45299 is added to blk_1073741830_1006 (size=10311)
2024-11-22T17:38:40,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35993 is added to blk_1073741830_1006 (size=10311)
2024-11-22T17:38:40,690 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-22T17:38:40,690 INFO  [M:0;e61df7af61d2:33881 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-22T17:38:40,690 INFO  [M:0;e61df7af61d2:33881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33881
2024-11-22T17:38:40,690 INFO  [M:0;e61df7af61d2:33881 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-22T17:38:40,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:40,800 INFO  [M:0;e61df7af61d2:33881 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-22T17:38:40,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33881-0x10163a04fc90000, quorum=127.0.0.1:56137, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-22T17:38:40,835 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@610f8a79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:40,836 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50ea3f07{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:40,836 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:40,837 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf154bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:40,837 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13f7f7ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:40,839 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:38:40,840 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:38:40,840 WARN  [BP-519871691-172.17.0.2-1732297116558 heartbeating to localhost/127.0.0.1:46159 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:38:40,840 WARN  [BP-519871691-172.17.0.2-1732297116558 heartbeating to localhost/127.0.0.1:46159 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-519871691-172.17.0.2-1732297116558 (Datanode Uuid 9ff0beb1-7ec8-4096-893a-dacf92478b00) service to localhost/127.0.0.1:46159
2024-11-22T17:38:40,841 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data3/current/BP-519871691-172.17.0.2-1732297116558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:40,842 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data4/current/BP-519871691-172.17.0.2-1732297116558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:40,842 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:38:40,848 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64cb4406{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-22T17:38:40,849 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27b8ac4b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:40,849 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:40,849 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7df47ec9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:40,849 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@676ee024{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:40,850 WARN  [BP-519871691-172.17.0.2-1732297116558 heartbeating to localhost/127.0.0.1:46159 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-22T17:38:40,850 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-22T17:38:40,850 WARN  [BP-519871691-172.17.0.2-1732297116558 heartbeating to localhost/127.0.0.1:46159 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-519871691-172.17.0.2-1732297116558 (Datanode Uuid c17b0a50-c86f-4cce-8d83-de443f6c42b9) service to localhost/127.0.0.1:46159
2024-11-22T17:38:40,850 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-22T17:38:40,850 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data1/current/BP-519871691-172.17.0.2-1732297116558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:40,851 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/cluster_79e38f68-2f76-99db-7b64-43280134a9ce/data/data2/current/BP-519871691-172.17.0.2-1732297116558 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-22T17:38:40,851 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-22T17:38:40,855 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@293ad0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-22T17:38:40,855 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@72a3013{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-22T17:38:40,855 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-22T17:38:40,856 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a903086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-22T17:38:40,856 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24e1a3ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/cab22fcc-f22d-24e8-c296-02707d8e8c88/hadoop.log.dir/,STOPPED}
2024-11-22T17:38:40,860 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-22T17:38:40,874 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-22T17:38:40,881 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 230)
Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46159
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46159 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46159
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46159
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46159 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:46159 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46159
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:46159
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=547 (was 524) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 179), ProcessCount=11 (was 11), AvailableMemoryMB=2406 (was 2417)