2024-11-10 02:07:23,296 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-10 02:07:23,310 main DEBUG Took 0.011747 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-10 02:07:23,310 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-10 02:07:23,311 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-10 02:07:23,312 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-10 02:07:23,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,322 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-10 02:07:23,335 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,338 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,339 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,340 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,341 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,343 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,344 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,345 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,346 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,346 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,347 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,348 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,349 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-10 02:07:23,350 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,351 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-10 02:07:23,353 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-10 02:07:23,354 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-10 02:07:23,356 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-10 02:07:23,357 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-10 02:07:23,359 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-10 02:07:23,359 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-10 02:07:23,368 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-10 02:07:23,371 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-10 02:07:23,374 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-10 02:07:23,374 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-10 02:07:23,375 main DEBUG createAppenders(={Console})
2024-11-10 02:07:23,375 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-10 02:07:23,376 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-10 02:07:23,376 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-10 02:07:23,377 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-10 02:07:23,377 main DEBUG OutputStream closed
2024-11-10 02:07:23,378 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-10 02:07:23,378 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-10 02:07:23,378 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-10 02:07:23,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-10 02:07:23,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-10 02:07:23,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-10 02:07:23,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-10 02:07:23,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-10 02:07:23,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-10 02:07:23,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-10 02:07:23,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-10 02:07:23,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-10 02:07:23,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-10 02:07:23,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-10 02:07:23,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-10 02:07:23,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-10 02:07:23,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-10 02:07:23,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-10 02:07:23,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-10 02:07:23,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-10 02:07:23,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-10 02:07:23,478 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-10 02:07:23,478 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-10 02:07:23,478 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-10 02:07:23,479 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-10T02:07:23,732 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952
2024-11-10 02:07:23,735 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-10 02:07:23,735 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-10T02:07:23,744 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-10T02:07:23,779 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=188, ProcessCount=11, AvailableMemoryMB=4291
2024-11-10T02:07:23,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:07:23,803 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7, deleteOnExit=true
2024-11-10T02:07:23,803 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:07:23,805 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/test.cache.data in system properties and HBase conf
2024-11-10T02:07:23,806 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:07:23,806 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:07:23,807 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:07:23,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:07:23,808 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:07:23,897 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-10T02:07:23,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:07:23,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:07:23,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:07:23,993 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:07:23,993 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:07:23,994 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:07:23,995 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:07:23,995 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:07:23,996 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:07:23,997 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:07:23,997 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:07:23,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:07:23,998 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:07:23,999 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:07:24,000 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:07:24,442 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:07:25,151 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-10T02:07:25,225 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2700ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-10T02:07:25,299 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:07:25,363 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:07:25,383 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:07:25,384 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:07:25,385 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:07:25,399 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:07:25,402 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:07:25,403 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:07:25,598 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/java.io.tmpdir/jetty-localhost-44961-hadoop-hdfs-3_4_1-tests_jar-_-any-13767996942111624683/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:07:25,610 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:44961}
2024-11-10T02:07:25,610 INFO  [Time-limited test {}] server.Server(415): Started @3086ms
2024-11-10T02:07:25,646 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:07:26,364 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:07:26,371 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:07:26,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:07:26,373 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:07:26,373 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:07:26,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:07:26,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:07:26,477 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/java.io.tmpdir/jetty-localhost-42707-hadoop-hdfs-3_4_1-tests_jar-_-any-2901322328253237905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:07:26,478 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:42707}
2024-11-10T02:07:26,478 INFO  [Time-limited test {}] server.Server(415): Started @3954ms
2024-11-10T02:07:26,539 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:07:26,676 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:07:26,681 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:07:26,682 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:07:26,682 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:07:26,682 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:07:26,684 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:07:26,684 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:07:26,811 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/java.io.tmpdir/jetty-localhost-40417-hadoop-hdfs-3_4_1-tests_jar-_-any-14727738907090842767/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:07:26,812 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:40417}
2024-11-10T02:07:26,812 INFO  [Time-limited test {}] server.Server(415): Started @4288ms
2024-11-10T02:07:26,815 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:07:27,913 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data4/current/BP-72610601-172.17.0.3-1731204444534/current, will proceed with Du for space computation calculation, 
2024-11-10T02:07:27,913 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data1/current/BP-72610601-172.17.0.3-1731204444534/current, will proceed with Du for space computation calculation, 
2024-11-10T02:07:27,913 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data2/current/BP-72610601-172.17.0.3-1731204444534/current, will proceed with Du for space computation calculation, 
2024-11-10T02:07:27,913 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data3/current/BP-72610601-172.17.0.3-1731204444534/current, will proceed with Du for space computation calculation, 
2024-11-10T02:07:27,957 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:07:27,958 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:07:28,009 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab6f7433caee169c with lease ID 0x5a23f3d4d0e19c3d: Processing first storage report for DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20 from datanode DatanodeRegistration(127.0.0.1:34485, datanodeUuid=2f007f08-99f3-403c-97af-8b1a31104f3a, infoPort=38669, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534)
2024-11-10T02:07:28,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab6f7433caee169c with lease ID 0x5a23f3d4d0e19c3d: from storage DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20 node DatanodeRegistration(127.0.0.1:34485, datanodeUuid=2f007f08-99f3-403c-97af-8b1a31104f3a, infoPort=38669, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-10T02:07:28,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb850902b541fe603 with lease ID 0x5a23f3d4d0e19c3e: Processing first storage report for DS-8dbc2373-e9be-4429-8790-cebd5bd4d663 from datanode DatanodeRegistration(127.0.0.1:39593, datanodeUuid=4f8d831b-f49d-4086-a7e9-3ad3144c5cce, infoPort=42495, infoSecurePort=0, ipcPort=41217, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534)
2024-11-10T02:07:28,012 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb850902b541fe603 with lease ID 0x5a23f3d4d0e19c3e: from storage DS-8dbc2373-e9be-4429-8790-cebd5bd4d663 node DatanodeRegistration(127.0.0.1:39593, datanodeUuid=4f8d831b-f49d-4086-a7e9-3ad3144c5cce, infoPort=42495, infoSecurePort=0, ipcPort=41217, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:07:28,012 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab6f7433caee169c with lease ID 0x5a23f3d4d0e19c3d: Processing first storage report for DS-f29210ad-5877-43a1-b0c9-a6542ca727fe from datanode DatanodeRegistration(127.0.0.1:34485, datanodeUuid=2f007f08-99f3-403c-97af-8b1a31104f3a, infoPort=38669, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534)
2024-11-10T02:07:28,013 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab6f7433caee169c with lease ID 0x5a23f3d4d0e19c3d: from storage DS-f29210ad-5877-43a1-b0c9-a6542ca727fe node DatanodeRegistration(127.0.0.1:34485, datanodeUuid=2f007f08-99f3-403c-97af-8b1a31104f3a, infoPort=38669, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:07:28,013 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb850902b541fe603 with lease ID 0x5a23f3d4d0e19c3e: Processing first storage report for DS-69800695-c684-4691-b5e0-f67da0e11223 from datanode DatanodeRegistration(127.0.0.1:39593, datanodeUuid=4f8d831b-f49d-4086-a7e9-3ad3144c5cce, infoPort=42495, infoSecurePort=0, ipcPort=41217, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534)
2024-11-10T02:07:28,013 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb850902b541fe603 with lease ID 0x5a23f3d4d0e19c3e: from storage DS-69800695-c684-4691-b5e0-f67da0e11223 node DatanodeRegistration(127.0.0.1:39593, datanodeUuid=4f8d831b-f49d-4086-a7e9-3ad3144c5cce, infoPort=42495, infoSecurePort=0, ipcPort=41217, storageInfo=lv=-57;cid=testClusterID;nsid=785782144;c=1731204444534), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:07:28,056 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952
2024-11-10T02:07:28,122 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/zookeeper_0, clientPort=62539, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:07:28,132 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62539
2024-11-10T02:07:28,145 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:28,149 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:28,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:07:28,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:07:28,806 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb with version=8
2024-11-10T02:07:28,807 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:07:28,883 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-10T02:07:29,212 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:07:29,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,221 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,226 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:07:29,226 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,226 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:07:29,372 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:07:29,425 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-10T02:07:29,434 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-10T02:07:29,437 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:07:29,459 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 102305 (auto-detected)
2024-11-10T02:07:29,460 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-10T02:07:29,477 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34977
2024-11-10T02:07:29,497 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34977 connecting to ZooKeeper ensemble=127.0.0.1:62539
2024-11-10T02:07:29,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349770x0, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:07:29,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34977-0x101227f7b150000 connected
2024-11-10T02:07:29,722 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:29,724 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:29,734 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:07:29,739 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb, hbase.cluster.distributed=false
2024-11-10T02:07:29,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:07:29,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34977
2024-11-10T02:07:29,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34977
2024-11-10T02:07:29,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34977
2024-11-10T02:07:29,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34977
2024-11-10T02:07:29,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34977
2024-11-10T02:07:29,903 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:07:29,905 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,905 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,906 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:07:29,906 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:07:29,906 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:07:29,910 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:07:29,913 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:07:29,914 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44507
2024-11-10T02:07:29,917 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44507 connecting to ZooKeeper ensemble=127.0.0.1:62539
2024-11-10T02:07:29,919 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:29,926 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:29,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445070x0, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:07:29,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445070x0, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:07:29,943 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44507-0x101227f7b150001 connected
2024-11-10T02:07:29,947 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:07:29,957 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:07:29,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:07:29,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:07:29,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44507
2024-11-10T02:07:29,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44507
2024-11-10T02:07:29,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44507
2024-11-10T02:07:29,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44507
2024-11-10T02:07:29,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44507
2024-11-10T02:07:29,990 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:34977
2024-11-10T02:07:29,991 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:07:30,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:07:30,006 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:07:30,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,036 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:07:30,038 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,34977,1731204449046 from backup master directory
2024-11-10T02:07:30,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:07:30,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:07:30,047 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:07:30,047 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,049 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-10T02:07:30,051 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-10T02:07:30,114 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase.id] with ID: 9b2dbc63-6164-42c1-ab31-b234c408fbe0
2024-11-10T02:07:30,114 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/.tmp/hbase.id
2024-11-10T02:07:30,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:07:30,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:07:30,128 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/.tmp/hbase.id]:[hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase.id]
2024-11-10T02:07:30,173 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:30,177 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:07:30,196 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-10T02:07:30,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:07:30,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:07:30,236 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:07:30,238 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:07:30,244 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:07:30,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:07:30,276 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:07:30,296 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store
2024-11-10T02:07:30,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:07:30,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:07:30,324 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-10T02:07:30,328 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:30,329 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:07:30,330 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:07:30,330 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:07:30,332 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:07:30,332 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:07:30,332 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:07:30,334 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204450329Disabling compacts and flushes for region at 1731204450329Disabling writes for close at 1731204450332 (+3 ms)Writing region close event to WAL at 1731204450332Closed at 1731204450332
2024-11-10T02:07:30,336 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/.initializing
2024-11-10T02:07:30,337 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/WALs/0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,360 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C34977%2C1731204449046, suffix=, logDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/WALs/0c4d4d227d60,34977,1731204449046, archiveDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/oldWALs, maxLogs=10
2024-11-10T02:07:30,372 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C34977%2C1731204449046.1731204450367
2024-11-10T02:07:30,397 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/WALs/0c4d4d227d60,34977,1731204449046/0c4d4d227d60%2C34977%2C1731204449046.1731204450367
2024-11-10T02:07:30,412 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-10T02:07:30,414 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:07:30,415 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:30,419 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,421 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:07:30,484 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:07:30,487 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:30,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:30,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:07:30,494 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:07:30,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:30,495 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:07:30,495 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:07:30,498 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:07:30,498 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:30,500 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:07:30,500 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:07:30,503 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:07:30,503 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:30,504 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:07:30,504 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,509 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,510 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,516 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,517 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,520 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:07:30,525 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:07:30,531 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:07:30,533 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=788539, jitterRate=0.0026804357767105103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:07:30,542 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204450436Initializing all the Stores at 1731204450439 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204450439Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204450440 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204450440Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204450440Cleaning up temporary data from old regions at 1731204450517 (+77 ms)Region opened successfully at 1731204450542 (+25 ms)
2024-11-10T02:07:30,544 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:07:30,580 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c65edb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:07:30,609 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:07:30,620 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:07:30,620 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:07:30,623 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:07:30,625 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-10T02:07:30,629 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-10T02:07:30,629 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:07:30,655 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:07:30,664 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:07:30,709 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:07:30,711 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:07:30,713 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:07:30,719 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:07:30,721 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:07:30,725 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:07:30,738 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:07:30,740 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:07:30,751 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:07:30,776 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:07:30,782 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:07:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:07:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:07:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,796 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,34977,1731204449046, sessionid=0x101227f7b150000, setting cluster-up flag (Was=false)
2024-11-10T02:07:30,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,856 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:07:30,858 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:30,909 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:07:30,911 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:30,916 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:07:30,978 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(746): ClusterId : 9b2dbc63-6164-42c1-ab31-b234c408fbe0
2024-11-10T02:07:30,982 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:07:30,984 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:07:30,994 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:07:30,996 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:07:30,996 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:07:31,001 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:07:31,005 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:07:31,006 DEBUG [RS:0;0c4d4d227d60:44507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40913b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:07:31,007 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,34977,1731204449046
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:07:31,015 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:07:31,016 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:07:31,016 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:07:31,016 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:07:31,016 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:07:31,016 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,017 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:07:31,017 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,021 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:44507
2024-11-10T02:07:31,025 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:07:31,025 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:07:31,025 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:07:31,025 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204481025
2024-11-10T02:07:31,027 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:07:31,027 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:07:31,027 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,34977,1731204449046 with port=44507, startcode=1731204449867
2024-11-10T02:07:31,028 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:07:31,028 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:07:31,031 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:07:31,032 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:07:31,032 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:07:31,033 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:07:31,035 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,035 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:07:31,036 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,042 DEBUG [RS:0;0c4d4d227d60:44507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:07:31,045 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:07:31,047 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:07:31,047 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:07:31,052 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:07:31,053 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:07:31,056 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204451054,5,FailOnTimeoutGroup]
2024-11-10T02:07:31,068 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204451057,5,FailOnTimeoutGroup]
2024-11-10T02:07:31,069 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,069 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:07:31,070 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,071 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:07:31,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:07:31,075 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:07:31,075 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb
2024-11-10T02:07:31,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:07:31,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:07:31,112 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:31,114 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:07:31,118 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:07:31,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:07:31,124 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:07:31,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,125 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,126 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:07:31,126 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60437, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:07:31,129 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:07:31,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,131 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:07:31,133 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34977 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,134 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:07:31,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,135 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,136 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:07:31,136 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34977 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,138 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740
2024-11-10T02:07:31,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740
2024-11-10T02:07:31,142 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:07:31,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:07:31,144 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:07:31,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:07:31,152 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:07:31,154 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb
2024-11-10T02:07:31,154 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36833
2024-11-10T02:07:31,154 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:07:31,154 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805142, jitterRate=0.023791596293449402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:07:31,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204451112Initializing all the Stores at 1731204451114 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451114Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204451114Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451114Cleaning up temporary data from old regions at 1731204451143 (+29 ms)Region opened successfully at 1731204451159 (+16 ms)
2024-11-10T02:07:31,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:07:31,160 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:07:31,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:07:31,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:07:31,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:07:31,162 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:07:31,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204451160Disabling compacts and flushes for region at 1731204451160Disabling writes for close at 1731204451160Writing region close event to WAL at 1731204451161 (+1 ms)Closed at 1731204451162 (+1 ms)
2024-11-10T02:07:31,165 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:07:31,166 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:07:31,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:07:31,170 DEBUG [RS:0;0c4d4d227d60:44507 {}] zookeeper.ZKUtil(111): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,171 WARN  [RS:0;0c4d4d227d60:44507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:07:31,171 INFO  [RS:0;0c4d4d227d60:44507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:07:31,171 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,173 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,44507,1731204449867]
2024-11-10T02:07:31,174 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:07:31,183 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:07:31,186 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:07:31,197 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:07:31,212 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:07:31,217 INFO  [RS:0;0c4d4d227d60:44507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:07:31,217 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,218 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:07:31,225 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:07:31,226 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,226 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:07:31,227 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:07:31,228 DEBUG [RS:0;0c4d4d227d60:44507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,230 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,44507,1731204449867-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:07:31,250 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:07:31,253 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,44507,1731204449867-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,253 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,253 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.Replication(171): 0c4d4d227d60,44507,1731204449867 started
2024-11-10T02:07:31,272 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:31,273 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,44507,1731204449867, RpcServer on 0c4d4d227d60/172.17.0.3:44507, sessionid=0x101227f7b150001
2024-11-10T02:07:31,273 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:07:31,274 DEBUG [RS:0;0c4d4d227d60:44507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,274 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,44507,1731204449867'
2024-11-10T02:07:31,274 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:07:31,275 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:07:31,276 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:07:31,276 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:07:31,276 DEBUG [RS:0;0c4d4d227d60:44507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,276 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,44507,1731204449867'
2024-11-10T02:07:31,276 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:07:31,277 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:07:31,278 DEBUG [RS:0;0c4d4d227d60:44507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:07:31,278 INFO  [RS:0;0c4d4d227d60:44507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:07:31,278 INFO  [RS:0;0c4d4d227d60:44507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:07:31,337 WARN  [0c4d4d227d60:34977 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:07:31,386 INFO  [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C44507%2C1731204449867, suffix=, logDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867, archiveDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs, maxLogs=32
2024-11-10T02:07:31,388 INFO  [RS:0;0c4d4d227d60:44507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204451388
2024-11-10T02:07:31,398 INFO  [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204451388
2024-11-10T02:07:31,401 DEBUG [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-10T02:07:31,589 DEBUG [0c4d4d227d60:34977 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:07:31,599 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,605 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,44507,1731204449867, state=OPENING
2024-11-10T02:07:31,656 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:07:31,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:31,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:07:31,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:07:31,668 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:07:31,669 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:07:31,671 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,44507,1731204449867}]
2024-11-10T02:07:31,846 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:07:31,850 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:07:31,861 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:07:31,861 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:07:31,865 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C44507%2C1731204449867.meta, suffix=.meta, logDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867, archiveDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs, maxLogs=32
2024-11-10T02:07:31,867 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.meta.1731204451867.meta
2024-11-10T02:07:31,875 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.meta.1731204451867.meta
2024-11-10T02:07:31,878 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:07:31,879 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:07:31,881 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:07:31,883 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:07:31,888 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:07:31,891 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:07:31,892 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:31,892 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:07:31,893 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:07:31,896 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:07:31,897 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:07:31,897 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,898 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,899 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:07:31,900 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:07:31,900 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,901 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,901 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:07:31,903 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:07:31,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,904 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,904 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:07:31,905 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:07:31,905 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:31,906 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:07:31,906 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:07:31,908 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740
2024-11-10T02:07:31,910 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740
2024-11-10T02:07:31,913 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:07:31,913 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:07:31,914 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:07:31,917 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:07:31,919 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794435, jitterRate=0.010176509618759155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:07:31,919 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:07:31,922 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204451893Writing region info on filesystem at 1731204451893Initializing all the Stores at 1731204451895 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451895Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451895Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204451895Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204451895Cleaning up temporary data from old regions at 1731204451913 (+18 ms)Running coprocessor post-open hooks at 1731204451919 (+6 ms)Region opened successfully at 1731204451921 (+2 ms)
2024-11-10T02:07:31,928 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204451838
2024-11-10T02:07:31,939 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:07:31,939 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:07:31,942 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:31,944 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,44507,1731204449867, state=OPEN
2024-11-10T02:07:32,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:07:32,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:07:32,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:07:32,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:07:32,080 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:32,087 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:07:32,087 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,44507,1731204449867 in 409 msec
2024-11-10T02:07:32,095 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:07:32,095 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 916 msec
2024-11-10T02:07:32,097 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:07:32,097 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:07:32,114 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:07:32,115 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,44507,1731204449867, seqNum=-1]
2024-11-10T02:07:32,132 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:07:32,133 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48051, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:07:32,152 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2070 sec
2024-11-10T02:07:32,152 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204452152, completionTime=-1
2024-11-10T02:07:32,155 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:07:32,155 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:07:32,182 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:07:32,182 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204512182
2024-11-10T02:07:32,182 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204572182
2024-11-10T02:07:32,183 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec
2024-11-10T02:07:32,185 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,186 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,186 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,187 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:34977, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,187 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,188 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,194 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:07:32,216 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.169sec
2024-11-10T02:07:32,217 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:07:32,219 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:07:32,220 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:07:32,220 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:07:32,220 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:07:32,221 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:07:32,222 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:07:32,229 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:07:32,230 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:07:32,231 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,34977,1731204449046-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:07:32,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:07:32,290 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-10T02:07:32,290 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-10T02:07:32,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,34977,-1 for getting cluster id
2024-11-10T02:07:32,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:07:32,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9b2dbc63-6164-42c1-ab31-b234c408fbe0'
2024-11-10T02:07:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:07:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9b2dbc63-6164-42c1-ab31-b234c408fbe0"

2024-11-10T02:07:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2461ffc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:07:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,34977,-1]
2024-11-10T02:07:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:07:32,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:07:32,313 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:07:32,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:07:32,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:07:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,44507,1731204449867, seqNum=-1]
2024-11-10T02:07:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:07:32,327 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:07:32,346 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:32,346 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:07:32,353 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:07:32,356 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-10T02:07:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c4d4d227d60,34977,1731204449046
2024-11-10T02:07:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c3c746e
2024-11-10T02:07:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-10T02:07:32,366 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-10T02:07:32,368 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-10T02:07:32,368 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-10T02:07:32,371 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:07:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-10T02:07:32,381 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-10T02:07:32,383 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-10T02:07:32,383 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:32,386 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-10T02:07:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:07:32,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741835_1011 (size=389)
2024-11-10T02:07:32,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741835_1011 (size=389)
2024-11-10T02:07:32,432 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df9f7697b39327b8714484a37e5727f8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb
2024-11-10T02:07:32,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741836_1012 (size=72)
2024-11-10T02:07:32,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741836_1012 (size=72)
2024-11-10T02:07:32,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:32,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing df9f7697b39327b8714484a37e5727f8, disabling compactions & flushes
2024-11-10T02:07:32,444 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,445 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,445 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8. after waiting 0 ms
2024-11-10T02:07:32,445 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,445 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,445 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for df9f7697b39327b8714484a37e5727f8:
Waiting for close lock at 1731204452444Disabling compacts and flushes for region at 1731204452444Disabling writes for close at 1731204452445 (+1 ms)Writing region close event to WAL at 1731204452445Closed at 1731204452445
2024-11-10T02:07:32,447 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-10T02:07:32,452 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731204452447"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204452447"}]},"ts":"1731204452447"}
2024-11-10T02:07:32,457 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-10T02:07:32,459 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-10T02:07:32,461 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204452459"}]},"ts":"1731204452459"}
2024-11-10T02:07:32,468 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-10T02:07:32,470 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=df9f7697b39327b8714484a37e5727f8, ASSIGN}]
2024-11-10T02:07:32,472 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=df9f7697b39327b8714484a37e5727f8, ASSIGN
2024-11-10T02:07:32,474 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=df9f7697b39327b8714484a37e5727f8, ASSIGN; state=OFFLINE, location=0c4d4d227d60,44507,1731204449867; forceNewPlan=false, retain=false
2024-11-10T02:07:32,626 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df9f7697b39327b8714484a37e5727f8, regionState=OPENING, regionLocation=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:32,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=df9f7697b39327b8714484a37e5727f8, ASSIGN because future has completed
2024-11-10T02:07:32,632 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df9f7697b39327b8714484a37e5727f8, server=0c4d4d227d60,44507,1731204449867}]
2024-11-10T02:07:32,793 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,794 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => df9f7697b39327b8714484a37e5727f8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:07:32,794 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,794 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:07:32,794 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,795 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,797 INFO  [StoreOpener-df9f7697b39327b8714484a37e5727f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region df9f7697b39327b8714484a37e5727f8 
2024-11-10T02:07:32,800 INFO  [StoreOpener-df9f7697b39327b8714484a37e5727f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df9f7697b39327b8714484a37e5727f8 columnFamilyName info
2024-11-10T02:07:32,800 DEBUG [StoreOpener-df9f7697b39327b8714484a37e5727f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:07:32,802 INFO  [StoreOpener-df9f7697b39327b8714484a37e5727f8-1 {}] regionserver.HStore(327): Store=df9f7697b39327b8714484a37e5727f8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:07:32,802 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,804 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,805 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,806 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,806 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,809 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,813 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:07:32,815 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened df9f7697b39327b8714484a37e5727f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=818086, jitterRate=0.040250495076179504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:07:32,815 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:32,817 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for df9f7697b39327b8714484a37e5727f8:
Running coprocessor pre-open hook at 1731204452795Writing region info on filesystem at 1731204452795Initializing all the Stores at 1731204452797 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204452797Cleaning up temporary data from old regions at 1731204452806 (+9 ms)Running coprocessor post-open hooks at 1731204452815 (+9 ms)Region opened successfully at 1731204452817 (+2 ms)
2024-11-10T02:07:32,819 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8., pid=6, masterSystemTime=1731204452787
2024-11-10T02:07:32,823 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,824 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:32,825 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=df9f7697b39327b8714484a37e5727f8, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,44507,1731204449867
2024-11-10T02:07:32,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure df9f7697b39327b8714484a37e5727f8, server=0c4d4d227d60,44507,1731204449867 because future has completed
2024-11-10T02:07:32,839 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-10T02:07:32,839 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure df9f7697b39327b8714484a37e5727f8, server=0c4d4d227d60,44507,1731204449867 in 202 msec
2024-11-10T02:07:32,844 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-10T02:07:32,844 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=df9f7697b39327b8714484a37e5727f8, ASSIGN in 369 msec
2024-11-10T02:07:32,846 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-10T02:07:32,847 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204452846"}]},"ts":"1731204452846"}
2024-11-10T02:07:32,851 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-10T02:07:32,853 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-10T02:07:32,859 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 479 msec
2024-11-10T02:07:37,401 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-10T02:07:37,449 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-10T02:07:37,451 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-10T02:07:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:07:39,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-10T02:07:39,424 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-10T02:07:39,424 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-10T02:07:39,425 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:07:39,425 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-10T02:07:39,426 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-10T02:07:39,426 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-10T02:07:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34977 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:07:42,422 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-10T02:07:42,424 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-10T02:07:42,431 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-10T02:07:42,431 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:07:42,432 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204462432
2024-11-10T02:07:42,442 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:07:42,442 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:07:42,442 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:07:42,442 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:07:42,443 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:07:42,443 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204451388 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204462432
2024-11-10T02:07:42,444 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-10T02:07:42,445 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204451388 is not closed yet, will try archiving it next time
2024-11-10T02:07:42,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741833_1009 (size=451)
2024-11-10T02:07:42,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741833_1009 (size=451)
2024-11-10T02:07:42,449 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204451388 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204451388
2024-11-10T02:07:42,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8., hostname=0c4d4d227d60,44507,1731204449867, seqNum=2]
2024-11-10T02:07:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44507 {}] regionserver.HRegion(8855): Flush requested on df9f7697b39327b8714484a37e5727f8
2024-11-10T02:07:54,491 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df9f7697b39327b8714484a37e5727f8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:07:54,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/1f2f42b21d3548db8849304705363a85 is 1080, key is row0001/info:/1731204462458/Put/seqid=0
2024-11-10T02:07:54,572 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741838_1014 (size=12509)
2024-11-10T02:07:54,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741838_1014 (size=12509)
2024-11-10T02:07:54,574 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/1f2f42b21d3548db8849304705363a85
2024-11-10T02:07:54,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/1f2f42b21d3548db8849304705363a85 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85
2024-11-10T02:07:54,636 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85, entries=7, sequenceid=11, filesize=12.2 K
2024-11-10T02:07:54,644 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 153ms, sequenceid=11, compaction requested=false
2024-11-10T02:07:54,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df9f7697b39327b8714484a37e5727f8:

2024-11-10T02:07:58,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:08:02,499 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204482499
2024-11-10T02:08:02,707 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:02,707 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:02,707 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:02,707 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:02,708 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:02,708 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:02,708 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204462432 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204482499
2024-11-10T02:08:02,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-10T02:08:02,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204462432 is not closed yet, will try archiving it next time
2024-11-10T02:08:02,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741837_1013 (size=12399)
2024-11-10T02:08:02,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741837_1013 (size=12399)
2024-11-10T02:08:02,912 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:05,116 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:07,320 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:09,524 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44507 {}] regionserver.HRegion(8855): Flush requested on df9f7697b39327b8714484a37e5727f8
2024-11-10T02:08:09,524 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df9f7697b39327b8714484a37e5727f8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:08:09,726 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:09,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/76a71d87bd3e4858a50cbaf89e27111c is 1080, key is row0008/info:/1731204476490/Put/seqid=0
2024-11-10T02:08:09,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741840_1016 (size=12509)
2024-11-10T02:08:09,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741840_1016 (size=12509)
2024-11-10T02:08:09,744 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/76a71d87bd3e4858a50cbaf89e27111c
2024-11-10T02:08:09,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/76a71d87bd3e4858a50cbaf89e27111c as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c
2024-11-10T02:08:09,767 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c, entries=7, sequenceid=21, filesize=12.2 K
2024-11-10T02:08:09,969 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:09,969 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 445ms, sequenceid=21, compaction requested=false
2024-11-10T02:08:09,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df9f7697b39327b8714484a37e5727f8:

2024-11-10T02:08:09,970 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-10T02:08:09,970 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:09,971 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85 because midkey is the same as first or last row
2024-11-10T02:08:11,728 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:12,234 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-10T02:08:12,234 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-10T02:08:13,932 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:13,933 WARN  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:13,934 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C44507%2C1731204449867:(num 1731204482499) roll requested
2024-11-10T02:08:13,935 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204493935
2024-11-10T02:08:14,142 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:14,143 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:14,143 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:14,143 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:14,143 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:14,143 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:14,143 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204482499 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204493935
2024-11-10T02:08:14,144 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:08:14,144 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204482499 is not closed yet, will try archiving it next time
2024-11-10T02:08:14,144 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204462432 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204462432
2024-11-10T02:08:14,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741839_1015 (size=7739)
2024-11-10T02:08:14,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741839_1015 (size=7739)
2024-11-10T02:08:16,136 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:17,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region df9f7697b39327b8714484a37e5727f8, had cached 0 bytes from a total of 25018
2024-11-10T02:08:18,340 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:20,544 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:22,748 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:24,750 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-10T02:08:24,751 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204504750
2024-11-10T02:08:28,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:08:29,759 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:29,760 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:29,760 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C44507%2C1731204449867:(num 1731204504750) roll requested
2024-11-10T02:08:29,761 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:29,761 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:29,761 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:29,761 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:29,761 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:29,761 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204493935 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204504750
2024-11-10T02:08:29,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741841_1017 (size=4753)
2024-11-10T02:08:29,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741841_1017 (size=4753)
2024-11-10T02:08:29,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38669:38669),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-10T02:08:29,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204493935 is not closed yet, will try archiving it next time
2024-11-10T02:08:29,773 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204509773
2024-11-10T02:08:34,777 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:34,777 WARN  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44507 {}] regionserver.HRegion(8855): Flush requested on df9f7697b39327b8714484a37e5727f8
2024-11-10T02:08:34,778 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df9f7697b39327b8714484a37e5727f8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:08:34,784 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:34,785 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:36,779 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-10T02:08:39,781 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:39,782 WARN  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK], DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK]]
2024-11-10T02:08:39,782 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:39,782 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:39,783 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:39,783 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:39,783 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:39,784 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204504750 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773
2024-11-10T02:08:39,786 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:08:39,786 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204504750 is not closed yet, will try archiving it next time
2024-11-10T02:08:39,786 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C44507%2C1731204449867:(num 1731204509773) roll requested
2024-11-10T02:08:39,787 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204519787
2024-11-10T02:08:39,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741842_1018 (size=1569)
2024-11-10T02:08:39,789 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741842_1018 (size=1569)
2024-11-10T02:08:39,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/7215590268d648d981a0edef6a58f7c2 is 1080, key is row0015/info:/1731204491526/Put/seqid=0
2024-11-10T02:08:39,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741844_1020 (size=12509)
2024-11-10T02:08:39,798 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741844_1020 (size=12509)
2024-11-10T02:08:39,798 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/7215590268d648d981a0edef6a58f7c2
2024-11-10T02:08:39,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/7215590268d648d981a0edef6a58f7c2 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2
2024-11-10T02:08:39,817 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2, entries=7, sequenceid=31, filesize=12.2 K
2024-11-10T02:08:44,797 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:44,797 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:44,818 INFO  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:44,818 WARN  [FSHLog-0-hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb-prefix:0c4d4d227d60,44507,1731204449867 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39593,DS-8dbc2373-e9be-4429-8790-cebd5bd4d663,DISK], DatanodeInfoWithStorage[127.0.0.1:34485,DS-7db0e790-8216-4bce-a7cc-9c0aca0cae20,DISK]]
2024-11-10T02:08:44,819 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,819 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 10040ms, sequenceid=31, compaction requested=true
2024-11-10T02:08:44,819 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df9f7697b39327b8714484a37e5727f8:

2024-11-10T02:08:44,819 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,819 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,819 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-10T02:08:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:44,819 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204519787
2024-11-10T02:08:44,819 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85 because midkey is the same as first or last row
2024-11-10T02:08:44,820 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:08:44,821 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,821 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204482499 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204482499
2024-11-10T02:08:44,821 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C44507%2C1731204449867:(num 1731204519787) roll requested
2024-11-10T02:08:44,821 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204524821
2024-11-10T02:08:44,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df9f7697b39327b8714484a37e5727f8:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:08:44,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741843_1019 (size=438)
2024-11-10T02:08:44,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741843_1019 (size=438)
2024-11-10T02:08:44,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:08:44,826 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:08:44,826 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204493935 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204493935
2024-11-10T02:08:44,828 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204504750 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204504750
2024-11-10T02:08:44,829 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:08:44,830 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HStore(1541): df9f7697b39327b8714484a37e5727f8/info is initiating minor compaction (all files)
2024-11-10T02:08:44,830 INFO  [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df9f7697b39327b8714484a37e5727f8/info in TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:08:44,831 INFO  [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2] into tmpdir=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp, totalSize=36.6 K
2024-11-10T02:08:44,832 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f2f42b21d3548db8849304705363a85, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731204462458
2024-11-10T02:08:44,833 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a71d87bd3e4858a50cbaf89e27111c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731204476490
2024-11-10T02:08:44,834 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7215590268d648d981a0edef6a58f7c2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731204491526
2024-11-10T02:08:44,844 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,844 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,844 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,844 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,845 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,845 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204519787 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204524821
2024-11-10T02:08:44,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741845_1021 (size=93)
2024-11-10T02:08:44,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741845_1021 (size=93)
2024-11-10T02:08:44,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:08:44,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204519787 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,851 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C44507%2C1731204449867.1731204524850
2024-11-10T02:08:44,864 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,865 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204519787 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204519787
2024-11-10T02:08:44,868 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,868 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,868 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,868 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,868 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:08:44,869 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204524821 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204524850
2024-11-10T02:08:44,871 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42495:42495),(127.0.0.1/127.0.0.1:38669:38669)]
2024-11-10T02:08:44,871 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,871 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204524821 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741846_1022 (size=1258)
2024-11-10T02:08:44,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741846_1022 (size=1258)
2024-11-10T02:08:44,875 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 is not closed yet, will try archiving it next time
2024-11-10T02:08:44,879 INFO  [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df9f7697b39327b8714484a37e5727f8#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:08:44,881 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/7c9f1550c1824353bbde97322954b363 is 1080, key is row0001/info:/1731204462458/Put/seqid=0
2024-11-10T02:08:44,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741848_1024 (size=27710)
2024-11-10T02:08:44,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741848_1024 (size=27710)
2024-11-10T02:08:44,904 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/7c9f1550c1824353bbde97322954b363 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7c9f1550c1824353bbde97322954b363
2024-11-10T02:08:44,924 INFO  [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df9f7697b39327b8714484a37e5727f8/info of df9f7697b39327b8714484a37e5727f8 into 7c9f1550c1824353bbde97322954b363(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:08:44,924 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df9f7697b39327b8714484a37e5727f8:

2024-11-10T02:08:44,926 INFO  [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8., storeName=df9f7697b39327b8714484a37e5727f8/info, priority=13, startTime=1731204524821; duration=0sec
2024-11-10T02:08:44,927 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-10T02:08:44,927 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:44,927 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7c9f1550c1824353bbde97322954b363 because midkey is the same as first or last row
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7c9f1550c1824353bbde97322954b363 because midkey is the same as first or last row
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7c9f1550c1824353bbde97322954b363 because midkey is the same as first or last row
2024-11-10T02:08:44,928 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:08:44,929 DEBUG [RS:0;0c4d4d227d60:44507-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df9f7697b39327b8714484a37e5727f8:info
2024-11-10T02:08:45,224 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/WALs/0c4d4d227d60,44507,1731204449867/0c4d4d227d60%2C44507%2C1731204449867.1731204509773 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs/0c4d4d227d60%2C44507%2C1731204449867.1731204509773
2024-11-10T02:08:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44507 {}] regionserver.HRegion(8855): Flush requested on df9f7697b39327b8714484a37e5727f8
2024-11-10T02:08:56,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df9f7697b39327b8714484a37e5727f8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:08:56,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/a3cb377fa10f48bcbebb118a27f72afb is 1080, key is row0022/info:/1731204524852/Put/seqid=0
2024-11-10T02:08:56,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741849_1025 (size=12509)
2024-11-10T02:08:56,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741849_1025 (size=12509)
2024-11-10T02:08:57,316 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/a3cb377fa10f48bcbebb118a27f72afb
2024-11-10T02:08:57,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/a3cb377fa10f48bcbebb118a27f72afb as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/a3cb377fa10f48bcbebb118a27f72afb
2024-11-10T02:08:57,349 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/a3cb377fa10f48bcbebb118a27f72afb, entries=7, sequenceid=42, filesize=12.2 K
2024-11-10T02:08:57,352 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 463ms, sequenceid=42, compaction requested=false
2024-11-10T02:08:57,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df9f7697b39327b8714484a37e5727f8:

2024-11-10T02:08:57,352 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-10T02:08:57,352 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:08:57,352 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7c9f1550c1824353bbde97322954b363 because midkey is the same as first or last row
2024-11-10T02:08:58,054 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:09:02,795 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region df9f7697b39327b8714484a37e5727f8, had cached 0 bytes from a total of 40219
2024-11-10T02:09:04,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:09:04,905 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:04,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:04,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:04,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:04,910 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:09:04,910 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:09:04,910 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=698085998, stopped=false
2024-11-10T02:09:04,910 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,34977,1731204449046
2024-11-10T02:09:04,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:04,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:04,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:04,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:04,923 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:09:04,923 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:04,923 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:04,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:04,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:04,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:04,924 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,44507,1731204449867' *****
2024-11-10T02:09:04,924 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:09:04,924 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:09:04,924 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:09:04,924 INFO  [RS:0;0c4d4d227d60:44507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:09:04,924 INFO  [RS:0;0c4d4d227d60:44507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(3091): Received CLOSE for df9f7697b39327b8714484a37e5727f8
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,44507,1731204449867
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:44507.
2024-11-10T02:09:04,925 DEBUG [RS:0;0c4d4d227d60:44507 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:04,925 DEBUG [RS:0;0c4d4d227d60:44507 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:09:04,925 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df9f7697b39327b8714484a37e5727f8, disabling compactions & flushes
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:09:04,925 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:09:04,925 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:09:04,926 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8. after waiting 0 ms
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:09:04,926 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:09:04,926 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df9f7697b39327b8714484a37e5727f8 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-10T02:09:04,926 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1325): Online Regions={df9f7697b39327b8714484a37e5727f8=TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8., 1588230740=hbase:meta,,1.1588230740}
2024-11-10T02:09:04,926 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:09:04,926 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:09:04,926 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, df9f7697b39327b8714484a37e5727f8
2024-11-10T02:09:04,926 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-10T02:09:04,932 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/fc73c351dc924a9299bf8d14dc68e291 is 1080, key is row0029/info:/1731204538891/Put/seqid=0
2024-11-10T02:09:04,945 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/info/b0d9fb2343b54f7181606757f855543d is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8./info:regioninfo/1731204452825/Put/seqid=0
2024-11-10T02:09:04,948 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741850_1026 (size=8193)
2024-11-10T02:09:04,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741850_1026 (size=8193)
2024-11-10T02:09:04,949 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/fc73c351dc924a9299bf8d14dc68e291
2024-11-10T02:09:04,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741851_1027 (size=7016)
2024-11-10T02:09:04,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741851_1027 (size=7016)
2024-11-10T02:09:04,955 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/info/b0d9fb2343b54f7181606757f855543d
2024-11-10T02:09:04,959 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/.tmp/info/fc73c351dc924a9299bf8d14dc68e291 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/fc73c351dc924a9299bf8d14dc68e291
2024-11-10T02:09:04,968 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/fc73c351dc924a9299bf8d14dc68e291, entries=3, sequenceid=48, filesize=8.0 K
2024-11-10T02:09:04,970 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 44ms, sequenceid=48, compaction requested=true
2024-11-10T02:09:04,970 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2] to archive
2024-11-10T02:09:04,974 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:09:04,978 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/archive/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/1f2f42b21d3548db8849304705363a85
2024-11-10T02:09:04,981 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/ns/89e7d2c5d15c4051bbbc4b7e6a067fa6 is 43, key is default/ns:d/1731204452137/Put/seqid=0
2024-11-10T02:09:04,981 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/archive/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/76a71d87bd3e4858a50cbaf89e27111c
2024-11-10T02:09:04,984 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2 to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/archive/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/info/7215590268d648d981a0edef6a58f7c2
2024-11-10T02:09:04,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741852_1028 (size=5153)
2024-11-10T02:09:04,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741852_1028 (size=5153)
2024-11-10T02:09:04,995 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0c4d4d227d60:34977 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-10T02:09:04,999 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1f2f42b21d3548db8849304705363a85=12509, 76a71d87bd3e4858a50cbaf89e27111c=12509, 7215590268d648d981a0edef6a58f7c2=12509]
2024-11-10T02:09:05,006 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/default/TestLogRolling-testSlowSyncLogRolling/df9f7697b39327b8714484a37e5727f8/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-10T02:09:05,008 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:09:05,008 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df9f7697b39327b8714484a37e5727f8:
Waiting for close lock at 1731204544925Running coprocessor pre-close hooks at 1731204544925Disabling compacts and flushes for region at 1731204544925Disabling writes for close at 1731204544926 (+1 ms)Obtaining lock to block concurrent updates at 1731204544926Preparing flush snapshotting stores in df9f7697b39327b8714484a37e5727f8 at 1731204544926Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731204544926Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8. at 1731204544927 (+1 ms)Flushing df9f7697b39327b8714484a37e5727f8/info: creating writer at 1731204544927Flushing df9f7697b39327b8714484a37e5727f8/info: appending metadata at 1731204544931 (+4 ms)Flushing df9f7697b39327b8714484a37e5727f8/info: closing flushed file at 1731204544932 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f15efcd: reopening flushed file at 1731204544958 (+26 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for df9f7697b39327b8714484a37e5727f8 in 44ms, sequenceid=48, compaction requested=true at 1731204544970 (+12 ms)Writing region close event to WAL at 1731204545000 (+30 ms)Running coprocessor post-close hooks at 1731204545006 (+6 ms)Closed at 1731204545008 (+2 ms)
2024-11-10T02:09:05,009 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731204452368.df9f7697b39327b8714484a37e5727f8.
2024-11-10T02:09:05,127 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-10T02:09:05,230 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:09:05,231 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:09:05,236 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:05,327 DEBUG [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-10T02:09:05,396 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/ns/89e7d2c5d15c4051bbbc4b7e6a067fa6
2024-11-10T02:09:05,423 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/table/c6a0e4020c024ae88264608ac71c249b is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731204452846/Put/seqid=0
2024-11-10T02:09:05,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741853_1029 (size=5396)
2024-11-10T02:09:05,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741853_1029 (size=5396)
2024-11-10T02:09:05,429 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/table/c6a0e4020c024ae88264608ac71c249b
2024-11-10T02:09:05,438 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/info/b0d9fb2343b54f7181606757f855543d as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/info/b0d9fb2343b54f7181606757f855543d
2024-11-10T02:09:05,447 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/info/b0d9fb2343b54f7181606757f855543d, entries=10, sequenceid=11, filesize=6.9 K
2024-11-10T02:09:05,449 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/ns/89e7d2c5d15c4051bbbc4b7e6a067fa6 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/ns/89e7d2c5d15c4051bbbc4b7e6a067fa6
2024-11-10T02:09:05,457 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/ns/89e7d2c5d15c4051bbbc4b7e6a067fa6, entries=2, sequenceid=11, filesize=5.0 K
2024-11-10T02:09:05,458 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/.tmp/table/c6a0e4020c024ae88264608ac71c249b as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/table/c6a0e4020c024ae88264608ac71c249b
2024-11-10T02:09:05,468 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/table/c6a0e4020c024ae88264608ac71c249b, entries=2, sequenceid=11, filesize=5.3 K
2024-11-10T02:09:05,470 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 543ms, sequenceid=11, compaction requested=false
2024-11-10T02:09:05,476 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-10T02:09:05,477 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:05,477 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:05,477 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204544926Running coprocessor pre-close hooks at 1731204544926Disabling compacts and flushes for region at 1731204544926Disabling writes for close at 1731204544926Obtaining lock to block concurrent updates at 1731204544926Preparing flush snapshotting stores in 1588230740 at 1731204544926Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731204544927 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731204544928 (+1 ms)Flushing 1588230740/info: creating writer at 1731204544928Flushing 1588230740/info: appending metadata at 1731204544945 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731204544945Flushing 1588230740/ns: creating writer at 1731204544962 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731204544981 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731204544981Flushing 1588230740/table: creating writer at 1731204545406 (+425 ms)Flushing 1588230740/table: appending metadata at 1731204545422 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731204545422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76923918: reopening flushed file at 1731204545437 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ed032f0: reopening flushed file at 1731204545447 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@631661cb: reopening flushed file at 1731204545457 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 543ms, sequenceid=11, compaction requested=false at 1731204545470 (+13 ms)Writing region close event to WAL at 1731204545471 (+1 ms)Running coprocessor post-close hooks at 1731204545476 (+5 ms)Closed at 1731204545477 (+1 ms)
2024-11-10T02:09:05,477 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:05,527 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,44507,1731204449867; all regions closed.
2024-11-10T02:09:05,529 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,530 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,530 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,530 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,530 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741834_1010 (size=3066)
2024-11-10T02:09:05,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741834_1010 (size=3066)
2024-11-10T02:09:05,538 DEBUG [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs
2024-11-10T02:09:05,538 INFO  [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C44507%2C1731204449867.meta:.meta(num 1731204451867)
2024-11-10T02:09:05,538 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,538 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,539 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,539 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,539 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741847_1023 (size=12695)
2024-11-10T02:09:05,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741847_1023 (size=12695)
2024-11-10T02:09:05,545 DEBUG [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/oldWALs
2024-11-10T02:09:05,545 INFO  [RS:0;0c4d4d227d60:44507 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C44507%2C1731204449867:(num 1731204524850)
2024-11-10T02:09:05,545 DEBUG [RS:0;0c4d4d227d60:44507 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:05,545 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:05,545 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:09:05,545 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-10T02:09:05,546 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:09:05,546 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:09:05,546 INFO  [RS:0;0c4d4d227d60:44507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44507
2024-11-10T02:09:05,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:05,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,44507,1731204449867
2024-11-10T02:09:05,554 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:09:05,565 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,44507,1731204449867]
2024-11-10T02:09:05,585 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,44507,1731204449867 already deleted, retry=false
2024-11-10T02:09:05,585 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,44507,1731204449867 expired; onlineServers=0
2024-11-10T02:09:05,586 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,34977,1731204449046' *****
2024-11-10T02:09:05,586 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:09:05,586 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:05,586 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:09:05,586 DEBUG [M:0;0c4d4d227d60:34977 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:09:05,586 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:09:05,586 DEBUG [M:0;0c4d4d227d60:34977 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:09:05,586 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204451057 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204451057,5,FailOnTimeoutGroup]
2024-11-10T02:09:05,586 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204451054 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204451054,5,FailOnTimeoutGroup]
2024-11-10T02:09:05,586 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:09:05,587 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:09:05,587 DEBUG [M:0;0c4d4d227d60:34977 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:09:05,587 INFO  [M:0;0c4d4d227d60:34977 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:09:05,587 INFO  [M:0;0c4d4d227d60:34977 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:09:05,587 INFO  [M:0;0c4d4d227d60:34977 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:09:05,587 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:09:05,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:09:05,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:05,596 DEBUG [M:0;0c4d4d227d60:34977 {}] zookeeper.ZKUtil(347): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:09:05,596 WARN  [M:0;0c4d4d227d60:34977 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:09:05,597 INFO  [M:0;0c4d4d227d60:34977 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/.lastflushedseqids
2024-11-10T02:09:05,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741854_1030 (size=130)
2024-11-10T02:09:05,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741854_1030 (size=130)
2024-11-10T02:09:05,609 INFO  [M:0;0c4d4d227d60:34977 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:09:05,609 INFO  [M:0;0c4d4d227d60:34977 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:09:05,609 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:09:05,609 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:05,609 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:05,609 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:09:05,609 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:05,610 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-10T02:09:05,628 DEBUG [M:0;0c4d4d227d60:34977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b046d9e04a84f999e532c51a9451cfe is 82, key is hbase:meta,,1/info:regioninfo/1731204451941/Put/seqid=0
2024-11-10T02:09:05,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741855_1031 (size=5672)
2024-11-10T02:09:05,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741855_1031 (size=5672)
2024-11-10T02:09:05,634 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b046d9e04a84f999e532c51a9451cfe
2024-11-10T02:09:05,662 DEBUG [M:0;0c4d4d227d60:34977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/151e741d7fe6493b92201e18dc10ea54 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731204452857/Put/seqid=0
2024-11-10T02:09:05,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:05,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44507-0x101227f7b150001, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:05,665 INFO  [RS:0;0c4d4d227d60:44507 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:09:05,666 INFO  [RS:0;0c4d4d227d60:44507 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,44507,1731204449867; zookeeper connection closed.
2024-11-10T02:09:05,666 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10b5d793 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10b5d793
2024-11-10T02:09:05,667 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:09:05,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741856_1032 (size=6248)
2024-11-10T02:09:05,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741856_1032 (size=6248)
2024-11-10T02:09:05,670 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/151e741d7fe6493b92201e18dc10ea54
2024-11-10T02:09:05,676 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 151e741d7fe6493b92201e18dc10ea54
2024-11-10T02:09:05,696 DEBUG [M:0;0c4d4d227d60:34977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9c32a0db7ed54fe98f21f38d78f83758 is 69, key is 0c4d4d227d60,44507,1731204449867/rs:state/1731204451139/Put/seqid=0
2024-11-10T02:09:05,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741857_1033 (size=5156)
2024-11-10T02:09:05,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741857_1033 (size=5156)
2024-11-10T02:09:05,703 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9c32a0db7ed54fe98f21f38d78f83758
2024-11-10T02:09:05,725 DEBUG [M:0;0c4d4d227d60:34977 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4c0a35bdc9594196b44bad11664dca01 is 52, key is load_balancer_on/state:d/1731204452350/Put/seqid=0
2024-11-10T02:09:05,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741858_1034 (size=5056)
2024-11-10T02:09:05,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741858_1034 (size=5056)
2024-11-10T02:09:05,732 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4c0a35bdc9594196b44bad11664dca01
2024-11-10T02:09:05,740 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b046d9e04a84f999e532c51a9451cfe as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b046d9e04a84f999e532c51a9451cfe
2024-11-10T02:09:05,748 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b046d9e04a84f999e532c51a9451cfe, entries=8, sequenceid=59, filesize=5.5 K
2024-11-10T02:09:05,749 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/151e741d7fe6493b92201e18dc10ea54 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/151e741d7fe6493b92201e18dc10ea54
2024-11-10T02:09:05,756 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 151e741d7fe6493b92201e18dc10ea54
2024-11-10T02:09:05,756 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/151e741d7fe6493b92201e18dc10ea54, entries=6, sequenceid=59, filesize=6.1 K
2024-11-10T02:09:05,757 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9c32a0db7ed54fe98f21f38d78f83758 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9c32a0db7ed54fe98f21f38d78f83758
2024-11-10T02:09:05,764 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9c32a0db7ed54fe98f21f38d78f83758, entries=1, sequenceid=59, filesize=5.0 K
2024-11-10T02:09:05,765 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4c0a35bdc9594196b44bad11664dca01 as hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4c0a35bdc9594196b44bad11664dca01
2024-11-10T02:09:05,771 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4c0a35bdc9594196b44bad11664dca01, entries=1, sequenceid=59, filesize=4.9 K
2024-11-10T02:09:05,773 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=59, compaction requested=false
2024-11-10T02:09:05,774 INFO  [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:05,774 DEBUG [M:0;0c4d4d227d60:34977 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204545609Disabling compacts and flushes for region at 1731204545609Disabling writes for close at 1731204545609Obtaining lock to block concurrent updates at 1731204545610 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204545610Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1731204545610Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204545611 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204545611Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204545628 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204545628Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204545641 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204545661 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204545662 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204545676 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204545695 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204545695Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204545710 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204545724 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204545724Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e5e6147: reopening flushed file at 1731204545739 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fbf5dfa: reopening flushed file at 1731204545748 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ba024ce: reopening flushed file at 1731204545756 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f258294: reopening flushed file at 1731204545764 (+8 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=59, compaction requested=false at 1731204545773 (+9 ms)Writing region close event to WAL at 1731204545774 (+1 ms)Closed at 1731204545774
2024-11-10T02:09:05,775 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,775 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,775 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,775 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,775 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:05,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39593 is added to blk_1073741830_1006 (size=27985)
2024-11-10T02:09:05,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34485 is added to blk_1073741830_1006 (size=27985)
2024-11-10T02:09:05,779 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:09:05,779 INFO  [M:0;0c4d4d227d60:34977 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:09:05,779 INFO  [M:0;0c4d4d227d60:34977 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34977
2024-11-10T02:09:05,779 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:09:05,886 INFO  [M:0;0c4d4d227d60:34977 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:09:05,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:05,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34977-0x101227f7b150000, quorum=127.0.0.1:62539, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:05,891 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:05,894 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:05,894 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:05,894 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:05,894 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:05,898 WARN  [BP-72610601-172.17.0.3-1731204444534 heartbeating to localhost/127.0.0.1:36833 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:05,898 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:05,898 WARN  [BP-72610601-172.17.0.3-1731204444534 heartbeating to localhost/127.0.0.1:36833 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-72610601-172.17.0.3-1731204444534 (Datanode Uuid 2f007f08-99f3-403c-97af-8b1a31104f3a) service to localhost/127.0.0.1:36833
2024-11-10T02:09:05,898 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:05,899 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data3/current/BP-72610601-172.17.0.3-1731204444534 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:05,899 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data4/current/BP-72610601-172.17.0.3-1731204444534 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:05,900 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:05,906 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:05,907 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:05,907 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:05,907 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:05,907 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:05,909 WARN  [BP-72610601-172.17.0.3-1731204444534 heartbeating to localhost/127.0.0.1:36833 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:05,909 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:05,909 WARN  [BP-72610601-172.17.0.3-1731204444534 heartbeating to localhost/127.0.0.1:36833 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-72610601-172.17.0.3-1731204444534 (Datanode Uuid 4f8d831b-f49d-4086-a7e9-3ad3144c5cce) service to localhost/127.0.0.1:36833
2024-11-10T02:09:05,909 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:05,910 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data1/current/BP-72610601-172.17.0.3-1731204444534 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:05,910 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/cluster_ba6fdfec-41d0-9da0-e1fc-efc3091fd6b7/data/data2/current/BP-72610601-172.17.0.3-1731204444534 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:05,911 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:05,924 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:09:05,925 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:05,925 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:05,925 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:05,925 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:05,935 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:09:05,967 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:09:05,978 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=79 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36833
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36833 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36833
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/0c4d4d227d60:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36833
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@7fef2b31
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36833
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36833
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36833 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0c4d4d227d60:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36833 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/0c4d4d227d60:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 188) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5863 (was 4291) - AvailableMemoryMB LEAK? -
2024-11-10T02:09:05,988 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=80, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=5863
2024-11-10T02:09:05,988 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:09:05,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.log.dir so I do NOT create it in target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4
2024-11-10T02:09:05,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4de2ba8e-cbe6-5984-6e43-189ad0bb2952/hadoop.tmp.dir so I do NOT create it in target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4
2024-11-10T02:09:05,989 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381, deleteOnExit=true
2024-11-10T02:09:05,989 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/test.cache.data in system properties and HBase conf
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:09:05,990 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:09:05,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:09:05,991 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:09:05,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:09:05,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:09:05,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:09:05,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:09:05,992 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:09:06,014 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:09:06,394 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:06,404 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:06,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:06,405 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:06,406 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:06,406 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:06,408 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:06,409 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:06,552 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51b7943b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/java.io.tmpdir/jetty-localhost-34425-hadoop-hdfs-3_4_1-tests_jar-_-any-6101144084131501603/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:09:06,553 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:34425}
2024-11-10T02:09:06,553 INFO  [Time-limited test {}] server.Server(415): Started @104029ms
2024-11-10T02:09:06,573 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:09:06,839 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:06,845 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:06,848 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:06,849 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:06,849 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:06,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:06,850 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:06,990 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b9e849{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/java.io.tmpdir/jetty-localhost-44987-hadoop-hdfs-3_4_1-tests_jar-_-any-13641251896751234183/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:06,990 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:44987}
2024-11-10T02:09:06,991 INFO  [Time-limited test {}] server.Server(415): Started @104466ms
2024-11-10T02:09:06,993 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:07,049 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:07,058 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:07,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:07,059 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:07,059 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:07,060 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:07,060 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:07,166 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e721f3d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/java.io.tmpdir/jetty-localhost-34199-hadoop-hdfs-3_4_1-tests_jar-_-any-13790134300605837524/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:07,166 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:34199}
2024-11-10T02:09:07,167 INFO  [Time-limited test {}] server.Server(415): Started @104642ms
2024-11-10T02:09:07,168 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:08,318 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data1/current/BP-1762703791-172.17.0.3-1731204546031/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:08,318 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data2/current/BP-1762703791-172.17.0.3-1731204546031/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:08,341 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:08,344 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc89dbeb049e26c14 with lease ID 0xcc3429869fa44692: Processing first storage report for DS-fea0c06b-a4f4-4519-ab0a-35ba6102930d from datanode DatanodeRegistration(127.0.0.1:41219, datanodeUuid=7bb07e04-1bcd-4f0d-ae01-3fb0bdedc55a, infoPort=41423, infoSecurePort=0, ipcPort=34395, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031)
2024-11-10T02:09:08,344 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc89dbeb049e26c14 with lease ID 0xcc3429869fa44692: from storage DS-fea0c06b-a4f4-4519-ab0a-35ba6102930d node DatanodeRegistration(127.0.0.1:41219, datanodeUuid=7bb07e04-1bcd-4f0d-ae01-3fb0bdedc55a, infoPort=41423, infoSecurePort=0, ipcPort=34395, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:08,345 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc89dbeb049e26c14 with lease ID 0xcc3429869fa44692: Processing first storage report for DS-9d784fa2-c99c-40e9-9e11-cb2dae212603 from datanode DatanodeRegistration(127.0.0.1:41219, datanodeUuid=7bb07e04-1bcd-4f0d-ae01-3fb0bdedc55a, infoPort=41423, infoSecurePort=0, ipcPort=34395, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031)
2024-11-10T02:09:08,345 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc89dbeb049e26c14 with lease ID 0xcc3429869fa44692: from storage DS-9d784fa2-c99c-40e9-9e11-cb2dae212603 node DatanodeRegistration(127.0.0.1:41219, datanodeUuid=7bb07e04-1bcd-4f0d-ae01-3fb0bdedc55a, infoPort=41423, infoSecurePort=0, ipcPort=34395, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:08,380 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data3/current/BP-1762703791-172.17.0.3-1731204546031/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:08,380 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data4/current/BP-1762703791-172.17.0.3-1731204546031/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:08,396 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:08,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xde4252b3bc616024 with lease ID 0xcc3429869fa44693: Processing first storage report for DS-651dc2dd-44b3-4a4c-8ced-7173c843fd6c from datanode DatanodeRegistration(127.0.0.1:35525, datanodeUuid=f6074853-ab42-4136-b902-200b2f7b38ac, infoPort=46441, infoSecurePort=0, ipcPort=33021, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031)
2024-11-10T02:09:08,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xde4252b3bc616024 with lease ID 0xcc3429869fa44693: from storage DS-651dc2dd-44b3-4a4c-8ced-7173c843fd6c node DatanodeRegistration(127.0.0.1:35525, datanodeUuid=f6074853-ab42-4136-b902-200b2f7b38ac, infoPort=46441, infoSecurePort=0, ipcPort=33021, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:08,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xde4252b3bc616024 with lease ID 0xcc3429869fa44693: Processing first storage report for DS-ad316091-5f3f-43c8-8d9b-122e6fabb0cc from datanode DatanodeRegistration(127.0.0.1:35525, datanodeUuid=f6074853-ab42-4136-b902-200b2f7b38ac, infoPort=46441, infoSecurePort=0, ipcPort=33021, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031)
2024-11-10T02:09:08,400 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xde4252b3bc616024 with lease ID 0xcc3429869fa44693: from storage DS-ad316091-5f3f-43c8-8d9b-122e6fabb0cc node DatanodeRegistration(127.0.0.1:35525, datanodeUuid=f6074853-ab42-4136-b902-200b2f7b38ac, infoPort=46441, infoSecurePort=0, ipcPort=33021, storageInfo=lv=-57;cid=testClusterID;nsid=137119945;c=1731204546031), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:08,412 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4
2024-11-10T02:09:08,435 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/zookeeper_0, clientPort=56306, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:09:08,436 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56306
2024-11-10T02:09:08,437 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,438 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:09:08,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:09:08,459 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69 with version=8
2024-11-10T02:09:08,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:09:08,461 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:09:08,461 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,461 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,462 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:09:08,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,462 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:09:08,462 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:09:08,462 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:09:08,463 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46757
2024-11-10T02:09:08,464 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46757 connecting to ZooKeeper ensemble=127.0.0.1:56306
2024-11-10T02:09:08,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:467570x0, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:09:08,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46757-0x101228102fd0000 connected
2024-11-10T02:09:08,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,614 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:08,618 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69, hbase.cluster.distributed=false
2024-11-10T02:09:08,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:09:08,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46757
2024-11-10T02:09:08,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46757
2024-11-10T02:09:08,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46757
2024-11-10T02:09:08,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46757
2024-11-10T02:09:08,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46757
2024-11-10T02:09:08,636 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:09:08,636 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:09:08,637 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:09:08,638 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43443
2024-11-10T02:09:08,639 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43443 connecting to ZooKeeper ensemble=127.0.0.1:56306
2024-11-10T02:09:08,640 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,642 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434430x0, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:09:08,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434430x0, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:08,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43443-0x101228102fd0001 connected
2024-11-10T02:09:08,654 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:09:08,655 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:09:08,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:09:08,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:09:08,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43443
2024-11-10T02:09:08,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43443
2024-11-10T02:09:08,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43443
2024-11-10T02:09:08,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43443
2024-11-10T02:09:08,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43443
2024-11-10T02:09:08,678 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:46757
2024-11-10T02:09:08,679 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:08,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:08,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:08,685 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:08,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:09:08,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,696 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:09:08,697 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,46757,1731204548461 from backup master directory
2024-11-10T02:09:08,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:08,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:08,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:08,706 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:09:08,706 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:08,711 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/hbase.id] with ID: 0f87f506-fbc3-4172-8460-e31edb119430
2024-11-10T02:09:08,711 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/.tmp/hbase.id
2024-11-10T02:09:08,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:09:08,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:09:08,719 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/.tmp/hbase.id]:[hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/hbase.id]
2024-11-10T02:09:08,732 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:08,732 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:09:08,733 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-10T02:09:08,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:09:08,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:09:08,755 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:09:08,756 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:09:08,756 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:08,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:09:08,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:09:08,765 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store
2024-11-10T02:09:08,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:09:08,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:09:08,774 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:08,774 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:08,774 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204548774Disabling compacts and flushes for region at 1731204548774Disabling writes for close at 1731204548774Writing region close event to WAL at 1731204548774Closed at 1731204548774
2024-11-10T02:09:08,775 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/.initializing
2024-11-10T02:09:08,775 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/WALs/0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:08,778 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C46757%2C1731204548461, suffix=, logDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/WALs/0c4d4d227d60,46757,1731204548461, archiveDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/oldWALs, maxLogs=10
2024-11-10T02:09:08,779 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46757%2C1731204548461.1731204548779
2024-11-10T02:09:08,784 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/WALs/0c4d4d227d60,46757,1731204548461/0c4d4d227d60%2C46757%2C1731204548461.1731204548779
2024-11-10T02:09:08,785 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41423:41423),(127.0.0.1/127.0.0.1:46441:46441)]
2024-11-10T02:09:08,786 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:09:08,786 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:08,786 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,786 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,788 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:08,789 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:09:08,790 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:08,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:08,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:08,792 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:09:08,792 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:08,793 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:08,793 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:08,795 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:09:08,795 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:08,796 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:08,796 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:08,798 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:09:08,798 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:08,798 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:08,798 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,799 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,800 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,801 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,802 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,802 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:09:08,804 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:08,806 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:09:08,807 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836592, jitterRate=0.06378231942653656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:09:08,808 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204548786Initializing all the Stores at 1731204548787 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204548787Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204548787Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204548787Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204548787Cleaning up temporary data from old regions at 1731204548802 (+15 ms)Region opened successfully at 1731204548808 (+6 ms)
2024-11-10T02:09:08,808 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:09:08,813 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136ade57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:09:08,814 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:09:08,814 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:09:08,814 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:09:08,814 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:09:08,815 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:09:08,816 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:09:08,816 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:09:08,819 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:09:08,820 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:09:08,832 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:09:08,832 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:09:08,833 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:09:08,842 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:09:08,843 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:09:08,844 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:09:08,853 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:09:08,854 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:09:08,864 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:09:08,867 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:09:08,874 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:09:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,885 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,46757,1731204548461, sessionid=0x101228102fd0000, setting cluster-up flag (Was=false)
2024-11-10T02:09:08,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:08,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:09,209 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:09:09,211 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:09,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:09,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:09,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:09,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:09:09,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-10T02:09:09,609 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:09:09,611 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:09,612 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:09:09,614 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:09,615 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:09:09,615 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:09:09,615 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,46757,1731204548461
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,617 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:09:09,618 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,621 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204579621
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:09:09,622 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:09:09,622 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:09:09,622 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:09:09,624 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:09,624 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,624 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:09:09,628 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:09:09,629 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:09:09,629 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:09:09,632 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:09:09,632 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:09:09,633 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204549633,5,FailOnTimeoutGroup]
2024-11-10T02:09:09,633 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204549633,5,FailOnTimeoutGroup]
2024-11-10T02:09:09,633 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,633 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:09:09,633 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,633 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:09:09,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:09:09,645 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:09:09,646 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69
2024-11-10T02:09:09,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:09:09,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:09:09,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:09,670 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(746): ClusterId : 0f87f506-fbc3-4172-8460-e31edb119430
2024-11-10T02:09:09,670 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:09:09,672 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:09:09,675 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:09:09,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:09,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:09,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:09:09,681 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:09:09,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:09,682 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:09,683 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:09:09,685 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:09:09,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:09,686 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:09,687 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:09:09,689 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:09:09,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:09,690 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:09,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:09:09,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740
2024-11-10T02:09:09,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740
2024-11-10T02:09:09,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:09:09,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:09:09,695 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:09:09,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:09:09,701 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:09:09,701 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=839242, jitterRate=0.06715230643749237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:09:09,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204549666Initializing all the Stores at 1731204549668 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204549668Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204549668Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204549668Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204549668Cleaning up temporary data from old regions at 1731204549694 (+26 ms)Region opened successfully at 1731204549703 (+9 ms)
2024-11-10T02:09:09,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:09:09,703 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:09:09,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:09:09,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:09:09,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:09:09,705 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:09,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204549703Disabling compacts and flushes for region at 1731204549703Disabling writes for close at 1731204549704 (+1 ms)Writing region close event to WAL at 1731204549705 (+1 ms)Closed at 1731204549705
2024-11-10T02:09:09,707 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:09,708 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:09:09,708 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:09:09,710 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:09:09,712 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:09:09,728 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:09:09,728 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:09:09,791 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:09:09,791 DEBUG [RS:0;0c4d4d227d60:43443 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aaf38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:09:09,804 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:43443
2024-11-10T02:09:09,804 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:09:09,805 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:09:09,805 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:09:09,806 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,46757,1731204548461 with port=43443, startcode=1731204548636
2024-11-10T02:09:09,806 DEBUG [RS:0;0c4d4d227d60:43443 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:09:09,809 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:09:09,810 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46757 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,811 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46757 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,815 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69
2024-11-10T02:09:09,815 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44293
2024-11-10T02:09:09,815 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:09:09,862 WARN  [0c4d4d227d60:46757 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:09:09,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:09,875 DEBUG [RS:0;0c4d4d227d60:43443 {}] zookeeper.ZKUtil(111): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,875 WARN  [RS:0;0c4d4d227d60:43443 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:09:09,875 INFO  [RS:0;0c4d4d227d60:43443 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:09,875 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/WALs/0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,875 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,43443,1731204548636]
2024-11-10T02:09:09,880 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:09:09,883 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:09:09,884 INFO  [RS:0;0c4d4d227d60:43443 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:09:09,884 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,884 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:09:09,886 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:09:09,886 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,886 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,886 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,886 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,886 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:09,887 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:09,888 DEBUG [RS:0;0c4d4d227d60:43443 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,890 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,43443,1731204548636-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:09:09,911 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:09:09,911 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,43443,1731204548636-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,912 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,912 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.Replication(171): 0c4d4d227d60,43443,1731204548636 started
2024-11-10T02:09:09,926 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:09,926 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,43443,1731204548636, RpcServer on 0c4d4d227d60/172.17.0.3:43443, sessionid=0x101228102fd0001
2024-11-10T02:09:09,926 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:09:09,926 DEBUG [RS:0;0c4d4d227d60:43443 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,926 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,43443,1731204548636'
2024-11-10T02:09:09,926 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:09:09,927 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,43443,1731204548636'
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:09:09,928 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:09:09,929 DEBUG [RS:0;0c4d4d227d60:43443 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:09:09,929 INFO  [RS:0;0c4d4d227d60:43443 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:09:09,929 INFO  [RS:0;0c4d4d227d60:43443 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:09:10,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:10,012 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:10,031 INFO  [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C43443%2C1731204548636, suffix=, logDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/WALs/0c4d4d227d60,43443,1731204548636, archiveDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/oldWALs, maxLogs=32
2024-11-10T02:09:10,033 INFO  [RS:0;0c4d4d227d60:43443 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C43443%2C1731204548636.1731204550033
2024-11-10T02:09:10,041 INFO  [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/WALs/0c4d4d227d60,43443,1731204548636/0c4d4d227d60%2C43443%2C1731204548636.1731204550033
2024-11-10T02:09:10,042 DEBUG [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46441:46441),(127.0.0.1/127.0.0.1:41423:41423)]
2024-11-10T02:09:10,112 DEBUG [0c4d4d227d60:46757 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:09:10,113 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:10,115 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,43443,1731204548636, state=OPENING
2024-11-10T02:09:10,295 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:09:10,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:10,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:10,306 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:09:10,307 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,43443,1731204548636}]
2024-11-10T02:09:10,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:10,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:10,461 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:09:10,463 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:09:10,467 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:09:10,467 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:10,470 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C43443%2C1731204548636.meta, suffix=.meta, logDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/WALs/0c4d4d227d60,43443,1731204548636, archiveDir=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/oldWALs, maxLogs=32
2024-11-10T02:09:10,472 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C43443%2C1731204548636.meta.1731204550472.meta
2024-11-10T02:09:10,478 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/WALs/0c4d4d227d60,43443,1731204548636/0c4d4d227d60%2C43443%2C1731204548636.meta.1731204550472.meta
2024-11-10T02:09:10,479 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41423:41423),(127.0.0.1/127.0.0.1:46441:46441)]
2024-11-10T02:09:10,480 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:09:10,480 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:09:10,481 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:09:10,481 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:09:10,481 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:09:10,481 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:10,481 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:09:10,481 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:09:10,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:09:10,485 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:09:10,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:10,485 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:10,485 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:09:10,486 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:09:10,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:10,487 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:10,487 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:09:10,488 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:09:10,488 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:10,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:10,489 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:09:10,490 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:09:10,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:10,490 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:10,490 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:09:10,492 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740
2024-11-10T02:09:10,493 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740
2024-11-10T02:09:10,495 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:09:10,495 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:09:10,496 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:09:10,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:09:10,499 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=702246, jitterRate=-0.10704921185970306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:09:10,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:09:10,500 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204550482Writing region info on filesystem at 1731204550482Initializing all the Stores at 1731204550483 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204550483Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204550483Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204550483Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204550483Cleaning up temporary data from old regions at 1731204550495 (+12 ms)Running coprocessor post-open hooks at 1731204550499 (+4 ms)Region opened successfully at 1731204550500 (+1 ms)
2024-11-10T02:09:10,502 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204550461
2024-11-10T02:09:10,504 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:09:10,505 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:09:10,505 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:10,507 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,43443,1731204548636, state=OPEN
2024-11-10T02:09:10,526 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:09:10,527 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:10,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:10,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:09:10,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:09:10,556 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:10,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:10,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:10,560 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:09:10,560 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,43443,1731204548636 in 250 msec
2024-11-10T02:09:10,563 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:09:10,563 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 852 msec
2024-11-10T02:09:10,564 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:10,564 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:09:10,566 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:09:10,566 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,43443,1731204548636, seqNum=-1]
2024-11-10T02:09:10,566 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:09:10,568 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:09:10,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 960 msec
2024-11-10T02:09:10,576 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204550575, completionTime=-1
2024-11-10T02:09:10,576 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:09:10,576 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:09:10,578 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:09:10,578 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204610578
2024-11-10T02:09:10,578 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204670578
2024-11-10T02:09:10,578 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:46757, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,582 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.880sec
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:09:10,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:09:10,587 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:09:10,590 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:09:10,590 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:09:10,590 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46757,1731204548461-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:10,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd69191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:10,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,46757,-1 for getting cluster id
2024-11-10T02:09:10,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:09:10,672 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f87f506-fbc3-4172-8460-e31edb119430'
2024-11-10T02:09:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:09:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f87f506-fbc3-4172-8460-e31edb119430"

2024-11-10T02:09:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa79d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,46757,-1]
2024-11-10T02:09:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:09:10,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,676 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:09:10,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e299140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:10,678 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:09:10,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,43443,1731204548636, seqNum=-1]
2024-11-10T02:09:10,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:09:10,683 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:09:10,685 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:10,686 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:10,690 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:09:10,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:09:10,690 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:10,690 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:10,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,691 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:09:10,691 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:09:10,691 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1972713769, stopped=false
2024-11-10T02:09:10,691 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,46757,1731204548461
2024-11-10T02:09:10,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:10,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:10,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:10,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:10,706 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:09:10,706 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:10,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:10,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:10,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:10,707 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,43443,1731204548636' *****
2024-11-10T02:09:10,707 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:09:10,708 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:43443.
2024-11-10T02:09:10,708 DEBUG [RS:0;0c4d4d227d60:43443 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:10,708 DEBUG [RS:0;0c4d4d227d60:43443 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:09:10,708 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:09:10,709 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:09:10,709 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-10T02:09:10,709 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-10T02:09:10,709 DEBUG [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-10T02:09:10,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:09:10,709 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:09:10,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:09:10,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:09:10,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:09:10,709 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-10T02:09:10,727 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/.tmp/ns/335a80fe39294dd3bbb60259fa4322e7 is 43, key is default/ns:d/1731204550569/Put/seqid=0
2024-11-10T02:09:10,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741835_1011 (size=5153)
2024-11-10T02:09:10,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741835_1011 (size=5153)
2024-11-10T02:09:10,733 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/.tmp/ns/335a80fe39294dd3bbb60259fa4322e7
2024-11-10T02:09:10,743 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/.tmp/ns/335a80fe39294dd3bbb60259fa4322e7 as hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/ns/335a80fe39294dd3bbb60259fa4322e7
2024-11-10T02:09:10,752 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/ns/335a80fe39294dd3bbb60259fa4322e7, entries=2, sequenceid=6, filesize=5.0 K
2024-11-10T02:09:10,754 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false
2024-11-10T02:09:10,754 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-10T02:09:10,761 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-10T02:09:10,761 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:10,762 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:10,762 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204550709Running coprocessor pre-close hooks at 1731204550709Disabling compacts and flushes for region at 1731204550709Disabling writes for close at 1731204550709Obtaining lock to block concurrent updates at 1731204550709Preparing flush snapshotting stores in 1588230740 at 1731204550709Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731204550710 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731204550711 (+1 ms)Flushing 1588230740/ns: creating writer at 1731204550711Flushing 1588230740/ns: appending metadata at 1731204550726 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731204550726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c41da66: reopening flushed file at 1731204550742 (+16 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=6, compaction requested=false at 1731204550754 (+12 ms)Writing region close event to WAL at 1731204550756 (+2 ms)Running coprocessor post-close hooks at 1731204550761 (+5 ms)Closed at 1731204550762 (+1 ms)
2024-11-10T02:09:10,762 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:10,894 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:09:10,894 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:09:10,909 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,43443,1731204548636; all regions closed.
2024-11-10T02:09:10,910 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,910 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,910 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,910 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,911 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741834_1010 (size=1152)
2024-11-10T02:09:10,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741834_1010 (size=1152)
2024-11-10T02:09:10,916 DEBUG [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/oldWALs
2024-11-10T02:09:10,916 INFO  [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C43443%2C1731204548636.meta:.meta(num 1731204550472)
2024-11-10T02:09:10,917 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,917 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,917 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,917 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,917 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:10,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741833_1009 (size=93)
2024-11-10T02:09:10,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741833_1009 (size=93)
2024-11-10T02:09:10,922 DEBUG [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/oldWALs
2024-11-10T02:09:10,922 INFO  [RS:0;0c4d4d227d60:43443 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C43443%2C1731204548636:(num 1731204550033)
2024-11-10T02:09:10,922 DEBUG [RS:0;0c4d4d227d60:43443 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:10,922 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:10,922 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:09:10,922 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-10T02:09:10,922 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:09:10,923 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:09:10,923 INFO  [RS:0;0c4d4d227d60:43443 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43443
2024-11-10T02:09:10,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:10,935 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:09:10,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,43443,1731204548636
2024-11-10T02:09:10,945 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,43443,1731204548636]
2024-11-10T02:09:10,956 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,43443,1731204548636 already deleted, retry=false
2024-11-10T02:09:10,956 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,43443,1731204548636 expired; onlineServers=0
2024-11-10T02:09:10,956 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,46757,1731204548461' *****
2024-11-10T02:09:10,956 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:09:10,956 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:10,956 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:09:10,956 DEBUG [M:0;0c4d4d227d60:46757 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:09:10,956 DEBUG [M:0;0c4d4d227d60:46757 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:09:10,956 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:09:10,956 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204549633 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204549633,5,FailOnTimeoutGroup]
2024-11-10T02:09:10,956 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204549633 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204549633,5,FailOnTimeoutGroup]
2024-11-10T02:09:10,956 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:09:10,957 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:09:10,957 DEBUG [M:0;0c4d4d227d60:46757 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:09:10,957 INFO  [M:0;0c4d4d227d60:46757 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:09:10,957 INFO  [M:0;0c4d4d227d60:46757 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:09:10,957 INFO  [M:0;0c4d4d227d60:46757 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:09:10,957 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:09:10,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:09:10,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:10,969 DEBUG [M:0;0c4d4d227d60:46757 {}] zookeeper.ZKUtil(347): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:09:10,969 WARN  [M:0;0c4d4d227d60:46757 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:09:10,970 INFO  [M:0;0c4d4d227d60:46757 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/.lastflushedseqids
2024-11-10T02:09:10,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741836_1012 (size=99)
2024-11-10T02:09:10,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741836_1012 (size=99)
2024-11-10T02:09:10,976 INFO  [M:0;0c4d4d227d60:46757 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:09:10,977 INFO  [M:0;0c4d4d227d60:46757 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:09:10,977 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:09:10,977 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:10,977 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:10,977 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:09:10,977 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:10,977 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-10T02:09:10,994 DEBUG [M:0;0c4d4d227d60:46757 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ce46238749c43c8959cbfeba2cdeb86 is 82, key is hbase:meta,,1/info:regioninfo/1731204550505/Put/seqid=0
2024-11-10T02:09:11,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741837_1013 (size=5672)
2024-11-10T02:09:11,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741837_1013 (size=5672)
2024-11-10T02:09:11,002 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ce46238749c43c8959cbfeba2cdeb86
2024-11-10T02:09:11,031 DEBUG [M:0;0c4d4d227d60:46757 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/04e896b0d655436d9c3f041c0f3ded8b is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731204550574/Put/seqid=0
2024-11-10T02:09:11,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741838_1014 (size=5275)
2024-11-10T02:09:11,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741838_1014 (size=5275)
2024-11-10T02:09:11,043 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/04e896b0d655436d9c3f041c0f3ded8b
2024-11-10T02:09:11,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:11,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43443-0x101228102fd0001, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:11,046 INFO  [RS:0;0c4d4d227d60:43443 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:09:11,046 INFO  [RS:0;0c4d4d227d60:43443 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,43443,1731204548636; zookeeper connection closed.
2024-11-10T02:09:11,046 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2dce5085 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2dce5085
2024-11-10T02:09:11,046 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:09:11,074 DEBUG [M:0;0c4d4d227d60:46757 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c4d89452f3b447409d8459c1b96d01cc is 69, key is 0c4d4d227d60,43443,1731204548636/rs:state/1731204549811/Put/seqid=0
2024-11-10T02:09:11,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741839_1015 (size=5156)
2024-11-10T02:09:11,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741839_1015 (size=5156)
2024-11-10T02:09:11,081 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c4d89452f3b447409d8459c1b96d01cc
2024-11-10T02:09:11,106 DEBUG [M:0;0c4d4d227d60:46757 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48e62c6daa724ca1ae4dbd77428f394f is 52, key is load_balancer_on/state:d/1731204550688/Put/seqid=0
2024-11-10T02:09:11,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741840_1016 (size=5056)
2024-11-10T02:09:11,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741840_1016 (size=5056)
2024-11-10T02:09:11,113 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48e62c6daa724ca1ae4dbd77428f394f
2024-11-10T02:09:11,124 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ce46238749c43c8959cbfeba2cdeb86 as hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ce46238749c43c8959cbfeba2cdeb86
2024-11-10T02:09:11,134 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ce46238749c43c8959cbfeba2cdeb86, entries=8, sequenceid=29, filesize=5.5 K
2024-11-10T02:09:11,136 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/04e896b0d655436d9c3f041c0f3ded8b as hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/04e896b0d655436d9c3f041c0f3ded8b
2024-11-10T02:09:11,144 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/04e896b0d655436d9c3f041c0f3ded8b, entries=3, sequenceid=29, filesize=5.2 K
2024-11-10T02:09:11,153 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c4d89452f3b447409d8459c1b96d01cc as hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c4d89452f3b447409d8459c1b96d01cc
2024-11-10T02:09:11,162 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c4d89452f3b447409d8459c1b96d01cc, entries=1, sequenceid=29, filesize=5.0 K
2024-11-10T02:09:11,163 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/48e62c6daa724ca1ae4dbd77428f394f as hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/48e62c6daa724ca1ae4dbd77428f394f
2024-11-10T02:09:11,173 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44293/user/jenkins/test-data/5026c8e7-35fc-5cad-9a11-eb1a848fce69/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/48e62c6daa724ca1ae4dbd77428f394f, entries=1, sequenceid=29, filesize=4.9 K
2024-11-10T02:09:11,174 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 197ms, sequenceid=29, compaction requested=false
2024-11-10T02:09:11,177 INFO  [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:11,177 DEBUG [M:0;0c4d4d227d60:46757 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204550977Disabling compacts and flushes for region at 1731204550977Disabling writes for close at 1731204550977Obtaining lock to block concurrent updates at 1731204550977Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204550977Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731204550978 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204550978Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204550978Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204550993 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204550993Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204551008 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204551030 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204551030Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204551053 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204551073 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204551074 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204551088 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204551105 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204551105Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f71928: reopening flushed file at 1731204551123 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76012891: reopening flushed file at 1731204551135 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a193964: reopening flushed file at 1731204551145 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a76b2f1: reopening flushed file at 1731204551162 (+17 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 197ms, sequenceid=29, compaction requested=false at 1731204551174 (+12 ms)Writing region close event to WAL at 1731204551177 (+3 ms)Closed at 1731204551177
2024-11-10T02:09:11,178 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:11,178 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:11,178 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:11,178 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:11,180 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:11,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35525 is added to blk_1073741830_1006 (size=10311)
2024-11-10T02:09:11,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41219 is added to blk_1073741830_1006 (size=10311)
2024-11-10T02:09:11,186 INFO  [M:0;0c4d4d227d60:46757 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:09:11,186 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:09:11,186 INFO  [M:0;0c4d4d227d60:46757 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46757
2024-11-10T02:09:11,186 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:09:11,298 INFO  [M:0;0c4d4d227d60:46757 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:09:11,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:11,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46757-0x101228102fd0000, quorum=127.0.0.1:56306, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:11,300 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e721f3d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:11,301 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:11,301 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:11,301 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:11,301 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:11,302 WARN  [BP-1762703791-172.17.0.3-1731204546031 heartbeating to localhost/127.0.0.1:44293 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:11,302 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:11,302 WARN  [BP-1762703791-172.17.0.3-1731204546031 heartbeating to localhost/127.0.0.1:44293 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1762703791-172.17.0.3-1731204546031 (Datanode Uuid f6074853-ab42-4136-b902-200b2f7b38ac) service to localhost/127.0.0.1:44293
2024-11-10T02:09:11,302 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:11,303 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data3/current/BP-1762703791-172.17.0.3-1731204546031 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:11,303 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data4/current/BP-1762703791-172.17.0.3-1731204546031 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:11,303 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:11,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b9e849{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:11,306 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:11,306 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:11,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:11,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:11,308 WARN  [BP-1762703791-172.17.0.3-1731204546031 heartbeating to localhost/127.0.0.1:44293 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:11,308 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:11,308 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:11,308 WARN  [BP-1762703791-172.17.0.3-1731204546031 heartbeating to localhost/127.0.0.1:44293 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1762703791-172.17.0.3-1731204546031 (Datanode Uuid 7bb07e04-1bcd-4f0d-ae01-3fb0bdedc55a) service to localhost/127.0.0.1:44293
2024-11-10T02:09:11,309 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data1/current/BP-1762703791-172.17.0.3-1731204546031 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:11,309 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/cluster_ba18940b-3ddf-0814-db54-b8a42f759381/data/data2/current/BP-1762703791-172.17.0.3-1731204546031 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:11,310 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:11,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51b7943b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:09:11,316 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:11,316 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:11,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:11,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:11,322 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.log.dir so I do NOT create it in target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e3099b12-c2df-7a6e-a510-9b88fedb09f4/hadoop.tmp.dir so I do NOT create it in target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b, deleteOnExit=true
2024-11-10T02:09:11,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/test.cache.data in system properties and HBase conf
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:09:11,342 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:09:11,342 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:09:11,343 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:09:11,344 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:09:11,364 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:09:11,719 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:11,729 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:11,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:11,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:11,731 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:09:11,732 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:11,733 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:11,734 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:11,881 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2295376c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-33747-hadoop-hdfs-3_4_1-tests_jar-_-any-10508474017435584971/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:09:11,882 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:33747}
2024-11-10T02:09:11,882 INFO  [Time-limited test {}] server.Server(415): Started @109357ms
2024-11-10T02:09:11,891 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:11,897 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:09:12,158 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:12,165 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:12,169 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:12,169 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:12,169 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:12,171 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@372d60ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:12,172 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a18c5e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:12,316 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bba803f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-44545-hadoop-hdfs-3_4_1-tests_jar-_-any-14635373858005464410/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:12,317 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7629a449{HTTP/1.1, (http/1.1)}{localhost:44545}
2024-11-10T02:09:12,317 INFO  [Time-limited test {}] server.Server(415): Started @109793ms
2024-11-10T02:09:12,319 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:12,379 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:12,385 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:12,385 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:12,386 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:12,386 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:09:12,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@217a95d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:12,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c64d82b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:12,530 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3efce601{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-43419-hadoop-hdfs-3_4_1-tests_jar-_-any-16347502173045972625/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:12,530 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e79a3d4{HTTP/1.1, (http/1.1)}{localhost:43419}
2024-11-10T02:09:12,530 INFO  [Time-limited test {}] server.Server(415): Started @110006ms
2024-11-10T02:09:12,533 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:13,645 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data1/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:13,646 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data2/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:13,673 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:13,676 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb640cda57ca7e4dd with lease ID 0x9fd12e564a2250d1: Processing first storage report for DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c from datanode DatanodeRegistration(127.0.0.1:41599, datanodeUuid=64335651-8813-4252-b8ed-798c9522c66b, infoPort=42981, infoSecurePort=0, ipcPort=33647, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:13,676 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb640cda57ca7e4dd with lease ID 0x9fd12e564a2250d1: from storage DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c node DatanodeRegistration(127.0.0.1:41599, datanodeUuid=64335651-8813-4252-b8ed-798c9522c66b, infoPort=42981, infoSecurePort=0, ipcPort=33647, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:13,676 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb640cda57ca7e4dd with lease ID 0x9fd12e564a2250d1: Processing first storage report for DS-18c1af06-176b-400f-9ae2-e46649494beb from datanode DatanodeRegistration(127.0.0.1:41599, datanodeUuid=64335651-8813-4252-b8ed-798c9522c66b, infoPort=42981, infoSecurePort=0, ipcPort=33647, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:13,677 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb640cda57ca7e4dd with lease ID 0x9fd12e564a2250d1: from storage DS-18c1af06-176b-400f-9ae2-e46649494beb node DatanodeRegistration(127.0.0.1:41599, datanodeUuid=64335651-8813-4252-b8ed-798c9522c66b, infoPort=42981, infoSecurePort=0, ipcPort=33647, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:13,794 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:13,794 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:13,825 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:13,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf13336aeb0a5b8bf with lease ID 0x9fd12e564a2250d2: Processing first storage report for DS-8220fa32-3509-4176-a4fd-14808e07cb0a from datanode DatanodeRegistration(127.0.0.1:43083, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=42421, infoSecurePort=0, ipcPort=33429, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:13,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf13336aeb0a5b8bf with lease ID 0x9fd12e564a2250d2: from storage DS-8220fa32-3509-4176-a4fd-14808e07cb0a node DatanodeRegistration(127.0.0.1:43083, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=42421, infoSecurePort=0, ipcPort=33429, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:09:13,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf13336aeb0a5b8bf with lease ID 0x9fd12e564a2250d2: Processing first storage report for DS-f7367f00-b920-4db9-b524-082b64362b51 from datanode DatanodeRegistration(127.0.0.1:43083, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=42421, infoSecurePort=0, ipcPort=33429, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:13,828 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf13336aeb0a5b8bf with lease ID 0x9fd12e564a2250d2: from storage DS-f7367f00-b920-4db9-b524-082b64362b51 node DatanodeRegistration(127.0.0.1:43083, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=42421, infoSecurePort=0, ipcPort=33429, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:13,889 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df
2024-11-10T02:09:13,892 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/zookeeper_0, clientPort=63612, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:09:13,893 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63612
2024-11-10T02:09:13,894 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:13,896 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:13,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:09:13,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:09:13,915 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7 with version=8
2024-11-10T02:09:13,915 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:09:13,917 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:09:13,917 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:09:13,918 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:09:13,919 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38021
2024-11-10T02:09:13,921 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38021 connecting to ZooKeeper ensemble=127.0.0.1:63612
2024-11-10T02:09:13,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380210x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:09:13,978 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38021-0x1012281184e0000 connected
2024-11-10T02:09:14,064 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:14,066 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:14,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:14,069 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7, hbase.cluster.distributed=false
2024-11-10T02:09:14,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:09:14,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38021
2024-11-10T02:09:14,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38021
2024-11-10T02:09:14,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38021
2024-11-10T02:09:14,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38021
2024-11-10T02:09:14,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38021
2024-11-10T02:09:14,095 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:09:14,095 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:09:14,096 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:09:14,097 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41989
2024-11-10T02:09:14,099 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41989 connecting to ZooKeeper ensemble=127.0.0.1:63612
2024-11-10T02:09:14,100 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:14,103 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:14,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419890x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:09:14,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419890x0, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:14,117 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41989-0x1012281184e0001 connected
2024-11-10T02:09:14,117 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:09:14,118 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:09:14,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:09:14,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:09:14,123 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41989
2024-11-10T02:09:14,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41989
2024-11-10T02:09:14,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41989
2024-11-10T02:09:14,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41989
2024-11-10T02:09:14,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41989
2024-11-10T02:09:14,145 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:38021
2024-11-10T02:09:14,145 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:14,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:14,159 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:09:14,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,170 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:09:14,170 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,38021,1731204553917 from backup master directory
2024-11-10T02:09:14,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:14,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,179 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:09:14,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:09:14,179 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,184 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/hbase.id] with ID: 69aa09ce-e474-4954-9655-f0a4eda84589
2024-11-10T02:09:14,184 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/.tmp/hbase.id
2024-11-10T02:09:14,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:09:14,192 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:09:14,192 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/.tmp/hbase.id]:[hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/hbase.id]
2024-11-10T02:09:14,207 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:14,207 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:09:14,209 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-10T02:09:14,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:09:14,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:09:14,230 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:09:14,232 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:09:14,232 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:14,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:09:14,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:09:14,248 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store
2024-11-10T02:09:14,259 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:09:14,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:09:14,261 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:14,261 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:09:14,262 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:14,262 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:14,262 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:09:14,262 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:14,262 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:09:14,262 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204554261Disabling compacts and flushes for region at 1731204554261Disabling writes for close at 1731204554262 (+1 ms)Writing region close event to WAL at 1731204554262Closed at 1731204554262
2024-11-10T02:09:14,264 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/.initializing
2024-11-10T02:09:14,264 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,268 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C38021%2C1731204553917, suffix=, logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917, archiveDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/oldWALs, maxLogs=10
2024-11-10T02:09:14,269 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C38021%2C1731204553917.1731204554269
2024-11-10T02:09:14,285 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269
2024-11-10T02:09:14,293 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42981:42981),(127.0.0.1/127.0.0.1:42421:42421)]
2024-11-10T02:09:14,296 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:09:14,296 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:14,297 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,297 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,300 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:14,303 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:09:14,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:14,304 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:14,306 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:09:14,306 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,307 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:14,307 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:14,310 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:09:14,310 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,311 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:14,311 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:09:14,322 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:09:14,322 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,323 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:14,323 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,324 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,325 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,327 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,327 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,328 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:09:14,330 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:09:14,338 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:09:14,339 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876065, jitterRate=0.11397545039653778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:09:14,340 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204554297Initializing all the Stores at 1731204554298 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204554298Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204554300 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204554300Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204554300Cleaning up temporary data from old regions at 1731204554327 (+27 ms)Region opened successfully at 1731204554340 (+13 ms)
2024-11-10T02:09:14,341 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:09:14,346 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aba7aa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:09:14,347 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:09:14,348 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:09:14,348 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:09:14,348 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:09:14,349 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:09:14,350 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:09:14,350 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:09:14,353 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:09:14,354 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:09:14,366 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:09:14,366 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:09:14,367 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:09:14,379 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:09:14,380 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:09:14,381 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:09:14,400 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:09:14,401 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:09:14,411 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:09:14,413 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:09:14,421 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:09:14,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:14,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:14,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,437 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,38021,1731204553917, sessionid=0x1012281184e0000, setting cluster-up flag (Was=false)
2024-11-10T02:09:14,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,495 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:09:14,497 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:14,548 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:09:14,549 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:14,551 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:09:14,553 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:14,553 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:09:14,554 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:09:14,554 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,38021,1731204553917
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:09:14,555 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:09:14,556 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,557 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204584557
2024-11-10T02:09:14,557 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:09:14,557 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:09:14,559 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:09:14,559 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:14,559 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:09:14,559 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:09:14,559 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:09:14,559 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:09:14,559 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204554559,5,FailOnTimeoutGroup]
2024-11-10T02:09:14,559 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204554559,5,FailOnTimeoutGroup]
2024-11-10T02:09:14,560 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,560 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:09:14,560 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,560 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,560 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,560 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:09:14,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:09:14,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:09:14,582 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:09:14,582 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7
2024-11-10T02:09:14,595 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:09:14,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:09:14,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:14,598 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:09:14,601 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:09:14,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,602 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:14,602 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:09:14,604 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:09:14,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,605 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:14,605 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:09:14,607 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:09:14,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,609 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:14,609 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:09:14,611 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:09:14,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:14,612 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:14,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:09:14,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740
2024-11-10T02:09:14,614 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740
2024-11-10T02:09:14,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:09:14,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:09:14,617 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:09:14,619 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:09:14,624 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:09:14,625 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=725398, jitterRate=-0.07760877907276154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:09:14,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204554596Initializing all the Stores at 1731204554598 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204554598Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204554598Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204554598Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204554598Cleaning up temporary data from old regions at 1731204554616 (+18 ms)Region opened successfully at 1731204554626 (+10 ms)
2024-11-10T02:09:14,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:09:14,626 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:09:14,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:09:14,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:09:14,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:09:14,627 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:14,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204554626Disabling compacts and flushes for region at 1731204554626Disabling writes for close at 1731204554626Writing region close event to WAL at 1731204554627 (+1 ms)Closed at 1731204554627
2024-11-10T02:09:14,629 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:14,629 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:09:14,630 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:09:14,632 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:09:14,641 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(746): ClusterId : 69aa09ce-e474-4954-9655-f0a4eda84589
2024-11-10T02:09:14,641 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:09:14,642 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:09:14,661 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:09:14,661 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:09:14,676 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:09:14,676 DEBUG [RS:0;0c4d4d227d60:41989 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4294e837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:09:14,700 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:41989
2024-11-10T02:09:14,700 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:09:14,700 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:09:14,700 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:09:14,701 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,38021,1731204553917 with port=41989, startcode=1731204554095
2024-11-10T02:09:14,702 DEBUG [RS:0;0c4d4d227d60:41989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:09:14,708 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50987, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:09:14,709 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,709 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38021 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,713 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7
2024-11-10T02:09:14,713 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37393
2024-11-10T02:09:14,713 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:09:14,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:14,728 DEBUG [RS:0;0c4d4d227d60:41989 {}] zookeeper.ZKUtil(111): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,728 WARN  [RS:0;0c4d4d227d60:41989 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:09:14,729 INFO  [RS:0;0c4d4d227d60:41989 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:14,729 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,735 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,41989,1731204554095]
2024-11-10T02:09:14,736 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:09:14,744 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:09:14,748 INFO  [RS:0;0c4d4d227d60:41989 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:09:14,749 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,749 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:09:14,752 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:09:14,752 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,752 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,752 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,753 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,754 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,754 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,754 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:14,754 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:14,754 DEBUG [RS:0;0c4d4d227d60:41989 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,757 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,41989,1731204554095-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:09:14,780 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:09:14,780 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,41989,1731204554095-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,781 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,781 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.Replication(171): 0c4d4d227d60,41989,1731204554095 started
2024-11-10T02:09:14,793 WARN  [0c4d4d227d60:38021 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:09:14,804 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:14,804 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,41989,1731204554095, RpcServer on 0c4d4d227d60/172.17.0.3:41989, sessionid=0x1012281184e0001
2024-11-10T02:09:14,804 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:09:14,804 DEBUG [RS:0;0c4d4d227d60:41989 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,804 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,41989,1731204554095'
2024-11-10T02:09:14,804 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:09:14,805 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:09:14,806 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:09:14,806 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:09:14,806 DEBUG [RS:0;0c4d4d227d60:41989 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:14,806 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,41989,1731204554095'
2024-11-10T02:09:14,806 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:09:14,810 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:09:14,811 DEBUG [RS:0;0c4d4d227d60:41989 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:09:14,811 INFO  [RS:0;0c4d4d227d60:41989 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:09:14,811 INFO  [RS:0;0c4d4d227d60:41989 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:09:14,914 INFO  [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C41989%2C1731204554095, suffix=, logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095, archiveDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs, maxLogs=32
2024-11-10T02:09:14,915 INFO  [RS:0;0c4d4d227d60:41989 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204554915
2024-11-10T02:09:14,926 INFO  [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915
2024-11-10T02:09:14,933 DEBUG [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42421:42421),(127.0.0.1/127.0.0.1:42981:42981)]
2024-11-10T02:09:15,043 DEBUG [0c4d4d227d60:38021 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:09:15,043 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:15,045 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,41989,1731204554095, state=OPENING
2024-11-10T02:09:15,179 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:09:15,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:15,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:15,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:15,191 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:09:15,191 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,41989,1731204554095}]
2024-11-10T02:09:15,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:15,346 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:09:15,348 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:09:15,353 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:09:15,353 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:15,356 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C41989%2C1731204554095.meta, suffix=.meta, logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095, archiveDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs, maxLogs=32
2024-11-10T02:09:15,357 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
2024-11-10T02:09:15,365 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
2024-11-10T02:09:15,366 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42421:42421),(127.0.0.1/127.0.0.1:42981:42981)]
2024-11-10T02:09:15,367 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:09:15,367 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:09:15,367 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:09:15,368 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:09:15,368 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:09:15,368 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:15,368 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:09:15,368 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:09:15,370 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:09:15,372 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:09:15,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:15,373 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:15,373 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:09:15,375 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:09:15,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:15,376 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:15,376 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:09:15,377 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:09:15,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:15,378 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:15,379 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:09:15,380 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:09:15,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:15,381 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:09:15,381 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:09:15,383 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740
2024-11-10T02:09:15,384 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740
2024-11-10T02:09:15,386 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:09:15,386 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:09:15,387 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:09:15,389 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:09:15,392 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728700, jitterRate=-0.07341109216213226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:09:15,392 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:09:15,393 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204555368Writing region info on filesystem at 1731204555368Initializing all the Stores at 1731204555370 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204555370Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204555370Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204555370Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204555370Cleaning up temporary data from old regions at 1731204555386 (+16 ms)Running coprocessor post-open hooks at 1731204555392 (+6 ms)Region opened successfully at 1731204555393 (+1 ms)
2024-11-10T02:09:15,395 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204555345
2024-11-10T02:09:15,399 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:09:15,399 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:09:15,401 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:15,402 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,41989,1731204554095, state=OPEN
2024-11-10T02:09:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:09:15,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:09:15,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:15,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:09:15,517 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:15,522 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:09:15,523 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,41989,1731204554095 in 326 msec
2024-11-10T02:09:15,528 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:09:15,528 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 894 msec
2024-11-10T02:09:15,529 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:09:15,529 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:09:15,532 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:09:15,532 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,41989,1731204554095, seqNum=-1]
2024-11-10T02:09:15,532 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:09:15,534 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41649, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:09:15,545 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 990 msec
2024-11-10T02:09:15,545 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204555545, completionTime=-1
2024-11-10T02:09:15,545 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:09:15,546 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:09:15,549 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:09:15,549 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204615549
2024-11-10T02:09:15,549 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204675549
2024-11-10T02:09:15,550 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:38021, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,564 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,568 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:09:15,571 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.392sec
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:09:15,572 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:09:15,576 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:09:15,576 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:09:15,576 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38021,1731204553917-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e18edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:15,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,38021,-1 for getting cluster id
2024-11-10T02:09:15,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:09:15,638 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '69aa09ce-e474-4954-9655-f0a4eda84589'
2024-11-10T02:09:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:09:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "69aa09ce-e474-4954-9655-f0a4eda84589"

2024-11-10T02:09:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dbbee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,38021,-1]
2024-11-10T02:09:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:09:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:15,643 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:09:15,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344f4078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:09:15,645 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:09:15,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,41989,1731204554095, seqNum=-1]
2024-11-10T02:09:15,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:09:15,654 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:09:15,657 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:15,658 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:15,663 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:09:15,690 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:09:15,691 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:09:15,727 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46049
2024-11-10T02:09:15,729 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46049 connecting to ZooKeeper ensemble=127.0.0.1:63612
2024-11-10T02:09:15,731 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:15,733 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:09:15,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460490x0, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:09:15,759 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-10T02:09:15,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46049-0x1012281184e0002 connected
2024-11-10T02:09:15,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-10T02:09:15,761 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:09:15,764 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:09:15,765 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:09:15,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:09:15,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46049
2024-11-10T02:09:15,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46049
2024-11-10T02:09:15,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46049
2024-11-10T02:09:15,776 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46049
2024-11-10T02:09:15,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46049
2024-11-10T02:09:15,779 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(746): ClusterId : 69aa09ce-e474-4954-9655-f0a4eda84589
2024-11-10T02:09:15,779 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:09:15,791 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:09:15,791 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:09:15,801 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:09:15,802 DEBUG [RS:1;0c4d4d227d60:46049 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae476a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:09:15,821 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0c4d4d227d60:46049
2024-11-10T02:09:15,821 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:09:15,821 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:09:15,821 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:09:15,822 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,38021,1731204553917 with port=46049, startcode=1731204555690
2024-11-10T02:09:15,823 DEBUG [RS:1;0c4d4d227d60:46049 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:09:15,825 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53791, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:09:15,825 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38021 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,826 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38021 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,828 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7
2024-11-10T02:09:15,828 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37393
2024-11-10T02:09:15,828 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:09:15,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:15,840 DEBUG [RS:1;0c4d4d227d60:46049 {}] zookeeper.ZKUtil(111): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,840 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,46049,1731204555690]
2024-11-10T02:09:15,840 WARN  [RS:1;0c4d4d227d60:46049 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:09:15,840 INFO  [RS:1;0c4d4d227d60:46049 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:09:15,841 DEBUG [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,844 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:09:15,854 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:09:15,854 INFO  [RS:1;0c4d4d227d60:46049 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:09:15,854 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,855 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:09:15,856 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:09:15,856 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:09:15,856 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:15,857 DEBUG [RS:1;0c4d4d227d60:46049 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,858 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46049,1731204555690-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:09:15,874 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:09:15,874 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46049,1731204555690-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,874 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,874 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.Replication(171): 0c4d4d227d60,46049,1731204555690 started
2024-11-10T02:09:15,890 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:09:15,890 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,46049,1731204555690, RpcServer on 0c4d4d227d60/172.17.0.3:46049, sessionid=0x1012281184e0002
2024-11-10T02:09:15,890 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:09:15,890 DEBUG [RS:1;0c4d4d227d60:46049 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,890 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,46049,1731204555690'
2024-11-10T02:09:15,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;0c4d4d227d60:46049,5,FailOnTimeoutGroup]
2024-11-10T02:09:15,890 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:09:15,891 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:09:15,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,46049,1731204555690'
2024-11-10T02:09:15,891 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:09:15,892 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:09:15,892 DEBUG [RS:1;0c4d4d227d60:46049 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:09:15,892 INFO  [RS:1;0c4d4d227d60:46049 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:09:15,892 INFO  [RS:1;0c4d4d227d60:46049 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:09:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dc13a3e
2024-11-10T02:09:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-10T02:09:15,898 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-10T02:09:15,899 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-10T02:09:15,899 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-10T02:09:15,899 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:09:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-10T02:09:15,902 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-10T02:09:15,902 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:15,902 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-10T02:09:15,904 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-10T02:09:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:09:15,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741835_1011 (size=393)
2024-11-10T02:09:15,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741835_1011 (size=393)
2024-11-10T02:09:15,919 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f26e09cb29c8f06b526c39c3f659f783, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7
2024-11-10T02:09:15,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43083 is added to blk_1073741836_1012 (size=76)
2024-11-10T02:09:15,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41599 is added to blk_1073741836_1012 (size=76)
2024-11-10T02:09:15,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:15,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing f26e09cb29c8f06b526c39c3f659f783, disabling compactions & flushes
2024-11-10T02:09:15,928 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:15,928 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:15,929 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783. after waiting 0 ms
2024-11-10T02:09:15,929 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:15,929 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:15,929 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for f26e09cb29c8f06b526c39c3f659f783:
Waiting for close lock at 1731204555928Disabling compacts and flushes for region at 1731204555928Disabling writes for close at 1731204555929 (+1 ms)Writing region close event to WAL at 1731204555929Closed at 1731204555929
2024-11-10T02:09:15,930 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-10T02:09:15,931 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731204555930"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204555930"}]},"ts":"1731204555930"}
2024-11-10T02:09:15,934 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-10T02:09:15,937 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-10T02:09:15,937 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204555937"}]},"ts":"1731204555937"}
2024-11-10T02:09:15,940 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-10T02:09:15,940 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f26e09cb29c8f06b526c39c3f659f783, ASSIGN}]
2024-11-10T02:09:15,942 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f26e09cb29c8f06b526c39c3f659f783, ASSIGN
2024-11-10T02:09:15,944 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f26e09cb29c8f06b526c39c3f659f783, ASSIGN; state=OFFLINE, location=0c4d4d227d60,41989,1731204554095; forceNewPlan=false, retain=false
2024-11-10T02:09:15,984 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:09:15,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:15,995 INFO  [RS:1;0c4d4d227d60:46049 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C46049%2C1731204555690, suffix=, logDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690, archiveDir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs, maxLogs=32
2024-11-10T02:09:15,996 INFO  [RS:1;0c4d4d227d60:46049 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46049%2C1731204555690.1731204555996
2024-11-10T02:09:16,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:16,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:16,014 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:16,029 INFO  [RS:1;0c4d4d227d60:46049 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
2024-11-10T02:09:16,030 DEBUG [RS:1;0c4d4d227d60:46049 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42981:42981),(127.0.0.1/127.0.0.1:42421:42421)]
2024-11-10T02:09:16,095 INFO  [0c4d4d227d60:38021 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-10T02:09:16,095 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f26e09cb29c8f06b526c39c3f659f783, regionState=OPENING, regionLocation=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:16,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f26e09cb29c8f06b526c39c3f659f783, ASSIGN because future has completed
2024-11-10T02:09:16,100 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f26e09cb29c8f06b526c39c3f659f783, server=0c4d4d227d60,41989,1731204554095}]
2024-11-10T02:09:16,258 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:16,258 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f26e09cb29c8f06b526c39c3f659f783, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:09:16,259 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,259 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:09:16,259 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,259 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,261 INFO  [StoreOpener-f26e09cb29c8f06b526c39c3f659f783-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f26e09cb29c8f06b526c39c3f659f783 
2024-11-10T02:09:16,263 INFO  [StoreOpener-f26e09cb29c8f06b526c39c3f659f783-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f26e09cb29c8f06b526c39c3f659f783 columnFamilyName info
2024-11-10T02:09:16,263 DEBUG [StoreOpener-f26e09cb29c8f06b526c39c3f659f783-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:09:16,263 INFO  [StoreOpener-f26e09cb29c8f06b526c39c3f659f783-1 {}] regionserver.HStore(327): Store=f26e09cb29c8f06b526c39c3f659f783/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:09:16,264 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,265 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,265 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,266 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,266 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,268 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,270 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:09:16,271 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f26e09cb29c8f06b526c39c3f659f783; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771287, jitterRate=-0.019258812069892883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:09:16,271 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:16,272 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f26e09cb29c8f06b526c39c3f659f783:
Running coprocessor pre-open hook at 1731204556260Writing region info on filesystem at 1731204556260Initializing all the Stores at 1731204556261 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204556261Cleaning up temporary data from old regions at 1731204556266 (+5 ms)Running coprocessor post-open hooks at 1731204556271 (+5 ms)Region opened successfully at 1731204556272 (+1 ms)
2024-11-10T02:09:16,274 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., pid=6, masterSystemTime=1731204556254
2024-11-10T02:09:16,277 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:16,277 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:16,278 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f26e09cb29c8f06b526c39c3f659f783, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:16,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f26e09cb29c8f06b526c39c3f659f783, server=0c4d4d227d60,41989,1731204554095 because future has completed
2024-11-10T02:09:16,294 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-10T02:09:16,294 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f26e09cb29c8f06b526c39c3f659f783, server=0c4d4d227d60,41989,1731204554095 in 186 msec
2024-11-10T02:09:16,297 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-10T02:09:16,298 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=f26e09cb29c8f06b526c39c3f659f783, ASSIGN in 354 msec
2024-11-10T02:09:16,301 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-10T02:09:16,301 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204556301"}]},"ts":"1731204556301"}
2024-11-10T02:09:16,304 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-10T02:09:16,306 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-10T02:09:16,309 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 407 msec
2024-11-10T02:09:19,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:09:19,422 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-10T02:09:19,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-10T02:09:19,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-10T02:09:19,424 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:19,424 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-10T02:09:20,737 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-10T02:09:21,762 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:09:21,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:21,785 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:21,789 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:21,790 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38021 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:09:26,002 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-10T02:09:26,002 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-10T02:09:26,005 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-10T02:09:26,005 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:26,020 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:26,023 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:26,024 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:26,025 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:26,025 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:26,025 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30008f24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:26,026 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ab5b96c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:26,145 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3fd17220{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-38757-hadoop-hdfs-3_4_1-tests_jar-_-any-12695018690918179737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:26,151 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c68f920{HTTP/1.1, (http/1.1)}{localhost:38757}
2024-11-10T02:09:26,151 INFO  [Time-limited test {}] server.Server(415): Started @123627ms
2024-11-10T02:09:26,153 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:26,214 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:26,219 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:26,220 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:26,220 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:26,220 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:09:26,221 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3f85c2b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:26,221 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64b89ed5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:26,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d82aaea{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-43219-hadoop-hdfs-3_4_1-tests_jar-_-any-4686641640660790251/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:26,324 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3904e150{HTTP/1.1, (http/1.1)}{localhost:43219}
2024-11-10T02:09:26,324 INFO  [Time-limited test {}] server.Server(415): Started @123800ms
2024-11-10T02:09:26,325 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:26,358 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:26,361 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:26,363 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:26,363 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:26,363 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:09:26,364 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b21f544{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:26,364 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47432b7b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:26,457 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20faceaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-46215-hadoop-hdfs-3_4_1-tests_jar-_-any-3265244917134837324/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:26,458 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e1ad43e{HTTP/1.1, (http/1.1)}{localhost:46215}
2024-11-10T02:09:26,458 INFO  [Time-limited test {}] server.Server(415): Started @123933ms
2024-11-10T02:09:26,459 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:27,597 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data5/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:27,597 WARN  [Thread-870 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data6/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:27,612 WARN  [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:27,614 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x93510f315864746 with lease ID 0x9fd12e564a2250d3: Processing first storage report for DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8 from datanode DatanodeRegistration(127.0.0.1:38203, datanodeUuid=33c7e553-8e49-411e-ba55-c6b2b7ac4505, infoPort=46201, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:27,614 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x93510f315864746 with lease ID 0x9fd12e564a2250d3: from storage DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8 node DatanodeRegistration(127.0.0.1:38203, datanodeUuid=33c7e553-8e49-411e-ba55-c6b2b7ac4505, infoPort=46201, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:27,614 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x93510f315864746 with lease ID 0x9fd12e564a2250d3: Processing first storage report for DS-dd260a9a-1f3f-4acb-bdd2-eee1155ab009 from datanode DatanodeRegistration(127.0.0.1:38203, datanodeUuid=33c7e553-8e49-411e-ba55-c6b2b7ac4505, infoPort=46201, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:27,614 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x93510f315864746 with lease ID 0x9fd12e564a2250d3: from storage DS-dd260a9a-1f3f-4acb-bdd2-eee1155ab009 node DatanodeRegistration(127.0.0.1:38203, datanodeUuid=33c7e553-8e49-411e-ba55-c6b2b7ac4505, infoPort=46201, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:27,763 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:27,763 WARN  [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:27,784 WARN  [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:27,787 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3dd682b6ef5ed70d with lease ID 0x9fd12e564a2250d4: Processing first storage report for DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567 from datanode DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:27,787 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3dd682b6ef5ed70d with lease ID 0x9fd12e564a2250d4: from storage DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567 node DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:27,787 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3dd682b6ef5ed70d with lease ID 0x9fd12e564a2250d4: Processing first storage report for DS-3c9057cc-1d1b-4486-964d-a3d0eeb1847c from datanode DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:27,787 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3dd682b6ef5ed70d with lease ID 0x9fd12e564a2250d4: from storage DS-3c9057cc-1d1b-4486-964d-a3d0eeb1847c node DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:09:28,258 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data9/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:28,258 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data10/current/BP-1233850745-172.17.0.3-1731204551379/current, will proceed with Du for space computation calculation, 
2024-11-10T02:09:28,279 WARN  [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:28,281 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb85a8fd9a24b780 with lease ID 0x9fd12e564a2250d5: Processing first storage report for DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca from datanode DatanodeRegistration(127.0.0.1:43909, datanodeUuid=48de2250-3df7-4a71-b3cd-674769b96cf4, infoPort=39021, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:28,282 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb85a8fd9a24b780 with lease ID 0x9fd12e564a2250d5: from storage DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca node DatanodeRegistration(127.0.0.1:43909, datanodeUuid=48de2250-3df7-4a71-b3cd-674769b96cf4, infoPort=39021, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:28,282 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb85a8fd9a24b780 with lease ID 0x9fd12e564a2250d5: Processing first storage report for DS-67224c0e-12c8-4773-9735-490f66fbdcd9 from datanode DatanodeRegistration(127.0.0.1:43909, datanodeUuid=48de2250-3df7-4a71-b3cd-674769b96cf4, infoPort=39021, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379)
2024-11-10T02:09:28,282 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb85a8fd9a24b780 with lease ID 0x9fd12e564a2250d5: from storage DS-67224c0e-12c8-4773-9735-490f66fbdcd9 node DatanodeRegistration(127.0.0.1:43909, datanodeUuid=48de2250-3df7-4a71-b3cd-674769b96cf4, infoPort=39021, infoSecurePort=0, ipcPort=42175, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:28,295 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,295 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,296 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta block BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:28,295 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,296 WARN  [PacketResponder: BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43083] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:28,296 WARN  [PacketResponder: BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43083] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_33516003_22 at /127.0.0.1:52336 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:41599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52336 dst: /127.0.0.1:41599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1040821705_22 at /127.0.0.1:52266 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52266 dst: /127.0.0.1:41599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:49172 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43083:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49172 dst: /127.0.0.1:43083
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:52298 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52298 dst: /127.0.0.1:41599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:49188 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43083:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49188 dst: /127.0.0.1:43083
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:52302 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41599:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52302 dst: /127.0.0.1:41599
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1040821705_22 at /127.0.0.1:49140 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43083:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49140 dst: /127.0.0.1:43083
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_33516003_22 at /127.0.0.1:49208 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:43083:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49208 dst: /127.0.0.1:43083
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:28,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3efce601{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:28,299 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e79a3d4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:28,299 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:28,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c64d82b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:28,300 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@217a95d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:28,300 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:28,300 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 29439ccc-285b-473b-a786-ec1b9ce6055d) service to localhost/127.0.0.1:37393
2024-11-10T02:09:28,301 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:28,304 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:28,304 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:28,304 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:28,304 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:28,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bba803f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:28,307 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7629a449{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:28,307 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:28,307 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a18c5e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:28,307 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@372d60ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:28,308 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:28,308 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:28,308 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 64335651-8813-4252-b8ed-798c9522c66b) service to localhost/127.0.0.1:37393
2024-11-10T02:09:28,308 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:28,309 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data1/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:28,309 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data2/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:28,309 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., hostname=0c4d4d227d60,41989,1731204554095, seqNum=2]
2024-11-10T02:09:28,322 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta block BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,322 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,322 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,322 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,323 WARN  [sync.2 {}] hdfs.DFSOutputStream(748): Error while syncing
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,323 ERROR [sync.2 {}] wal.FSHLog$SyncRunner(463): Error syncing
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,323 WARN  [FSHLog-0-hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7-prefix:0c4d4d227d60,41989,1731204554095 {}] wal.AbstractFSWAL(1504): sync failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,323 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,323 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C41989%2C1731204554095:(num 1731204554915) roll requested
2024-11-10T02:09:28,324 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204568323
2024-11-10T02:09:28,329 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:28,329 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:28,329 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:28,329 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:28,329 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:28,329 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204568323
2024-11-10T02:09:28,330 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,330 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:28,330 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39021:39021),(127.0.0.1/127.0.0.1:46201:46201)]
2024-11-10T02:09:28,330 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:28,331 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-10T02:09:28,331 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-10T02:09:28,331 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915
2024-11-10T02:09:28,334 WARN  [IPC Server handler 4 on default port 37393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-10T02:09:28,337 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 after 4ms
2024-11-10T02:09:29,858 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:30,233 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:30,330 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:30,332 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204568323
2024-11-10T02:09:30,332 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:30,333 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204568323 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:30,333 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:37244 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:43909:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37244 dst: /127.0.0.1:43909
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:30,334 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:48804 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:38203:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48804 dst: /127.0.0.1:38203
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:30,385 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20faceaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:30,385 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e1ad43e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:30,385 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:30,385 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47432b7b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:30,385 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b21f544{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:30,386 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:30,386 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:30,387 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 48de2250-3df7-4a71-b3cd-674769b96cf4) service to localhost/127.0.0.1:37393
2024-11-10T02:09:30,387 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:30,387 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data9/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:30,387 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data10/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:30,387 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:31,859 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:32,234 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:32,331 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]
2024-11-10T02:09:32,331 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:32,332 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C41989%2C1731204554095:(num 1731204568323) roll requested
2024-11-10T02:09:32,332 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204572332
2024-11-10T02:09:32,337 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:32,337 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35624 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:32,337 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:32,337 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021
2024-11-10T02:09:32,337 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35624 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-10T02:09:32,337 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35624 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35624 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:32,339 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 after 4008ms
2024-11-10T02:09:32,340 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:32,345 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43083
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:32,345 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35638 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022 to mirror 127.0.0.1:43083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:32,345 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:32,346 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022
2024-11-10T02:09:32,346 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35638 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-10T02:09:32,346 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35638 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35638 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:32,346 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:32,356 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:32,359 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:32,359 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:32,359 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:32,359 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:32,360 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204568323 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204572332
2024-11-10T02:09:32,362 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38203 is added to blk_1073741838_1020 (size=2431)
2024-11-10T02:09:32,363 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:32,368 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46201:46201),(127.0.0.1/127.0.0.1:45991:45991)]
2024-11-10T02:09:32,368 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:32,395 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-10T02:09:33,627 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@644ef4dd[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38203, datanodeUuid=33c7e553-8e49-411e-ba55-c6b2b7ac4505, infoPort=46201, infoSecurePort=0, ipcPort=41151, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741838_1020 to 127.0.0.1:43909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:33,859 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,234 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,369 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,399 WARN  [ResponseProcessor for block BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,400 WARN  [DataStreamer for file /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204572332 block BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:34,400 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:56392 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:38203:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56392 dst: /127.0.0.1:38203
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:34,400 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35644 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35644 dst: /127.0.0.1:37331
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:34,455 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3fd17220{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:34,455 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c68f920{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:09:34,455 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:09:34,455 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ab5b96c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:09:34,455 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30008f24{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:09:34,457 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:09:34,457 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:09:34,457 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:09:34,457 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 33c7e553-8e49-411e-ba55-c6b2b7ac4505) service to localhost/127.0.0.1:37393
2024-11-10T02:09:34,458 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data5/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:34,458 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data6/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:09:34,458 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:09:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:34,470 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:09:34,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/468f5a42792143cca614cd863bfa281e is 1080, key is row0002/info:/1731204570389/Put/seqid=0
2024-11-10T02:09:34,508 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,508 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:34,508 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741842_1025
2024-11-10T02:09:34,509 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:34,510 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,511 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:34,511 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741843_1026
2024-11-10T02:09:34,512 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:34,516 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35664 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:34,516 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:34,516 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027
2024-11-10T02:09:34,516 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35664 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:34,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35664 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35664 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:34,520 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:34,522 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:34,522 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:34,522 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741845_1028
2024-11-10T02:09:34,523 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:34,524 WARN  [IPC Server handler 1 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:34,524 WARN  [IPC Server handler 1 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:34,524 WARN  [IPC Server handler 1 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:34,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741846_1029 (size=10347)
2024-11-10T02:09:34,929 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/468f5a42792143cca614cd863bfa281e
2024-11-10T02:09:34,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/468f5a42792143cca614cd863bfa281e as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e
2024-11-10T02:09:34,943 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e, entries=5, sequenceid=11, filesize=10.1 K
2024-11-10T02:09:34,944 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for f26e09cb29c8f06b526c39c3f659f783 in 474ms, sequenceid=11, compaction requested=false
2024-11-10T02:09:34,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:35,105 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-10T02:09:35,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5c2be0310cd44829a2b0b20d869eaf98 is 1080, key is row0007/info:/1731204574472/Put/seqid=0
2024-11-10T02:09:35,113 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:35,113 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:35,113 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741847_1030
2024-11-10T02:09:35,114 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:35,115 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:35,116 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:35,116 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741848_1031
2024-11-10T02:09:35,117 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:35,118 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:35,119 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:35,119 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741849_1032
2024-11-10T02:09:35,119 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:35,122 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:35,122 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35696 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:35,122 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:35,122 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033
2024-11-10T02:09:35,122 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35696 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:35,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35696 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35696 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:35,123 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:35,124 WARN  [IPC Server handler 3 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:35,124 WARN  [IPC Server handler 3 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:35,124 WARN  [IPC Server handler 3 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:35,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741851_1034 (size=12506)
2024-11-10T02:09:35,528 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5c2be0310cd44829a2b0b20d869eaf98
2024-11-10T02:09:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5c2be0310cd44829a2b0b20d869eaf98 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98
2024-11-10T02:09:35,540 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98, entries=7, sequenceid=24, filesize=12.2 K
2024-11-10T02:09:35,541 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for f26e09cb29c8f06b526c39c3f659f783 in 436ms, sequenceid=24, compaction requested=false
2024-11-10T02:09:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-10T02:09:35,541 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:35,542 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98 because midkey is the same as first or last row
2024-11-10T02:09:35,859 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,235 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,369 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]
2024-11-10T02:09:36,369 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,369 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C41989%2C1731204554095:(num 1731204572332) roll requested
2024-11-10T02:09:36,370 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204576369
2024-11-10T02:09:36,373 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,374 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:36,374 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741852_1035
2024-11-10T02:09:36,374 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:36,376 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,376 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:36,376 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741853_1036
2024-11-10T02:09:36,377 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:36,379 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,379 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:36,379 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741854_1037
2024-11-10T02:09:36,380 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:36,383 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43083
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,383 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35702 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038 to mirror 127.0.0.1:43083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,383 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:36,383 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038
2024-11-10T02:09:36,383 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35702 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-10T02:09:36,383 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35702 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35702 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,386 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:36,387 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:36,387 WARN  [IPC Server handler 2 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:36,387 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:36,394 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:36,394 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:36,394 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:36,394 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:36,394 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:36,395 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204572332 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204576369
2024-11-10T02:09:36,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741841_1024 (size=25992)
2024-11-10T02:09:36,397 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45991:45991)]
2024-11-10T02:09:36,397 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:36,397 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204572332 is not closed yet, will try archiving it next time
2024-11-10T02:09:36,398 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204568323 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs/0c4d4d227d60%2C41989%2C1731204554095.1731204568323
2024-11-10T02:09:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:36,525 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-10T02:09:36,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/078c231c79ed4a7bb35e249abd3bea04 is 1079, key is tmprow/info:/1731204576524/Put/seqid=0
2024-11-10T02:09:36,534 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41599
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,534 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35710 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040 to mirror 127.0.0.1:41599
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,535 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:36,535 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35710 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:36,535 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040
2024-11-10T02:09:36,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35710 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35710 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,536 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:36,539 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,539 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35716 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,539 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:36,539 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35716 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:36,539 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041
2024-11-10T02:09:36,539 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35716 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35716 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,540 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:36,542 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,542 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:36,542 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741859_1042
2024-11-10T02:09:36,543 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:36,546 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:36,546 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35732 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,546 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:36,546 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043
2024-11-10T02:09:36,546 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35732 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:36,546 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35732 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35732 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,547 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:36,548 WARN  [IPC Server handler 1 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:36,549 WARN  [IPC Server handler 1 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:36,549 WARN  [IPC Server handler 1 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:36,554 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741861_1044 (size=6027)
2024-11-10T02:09:36,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@edd0c89[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741851_1034 to 127.0.0.1:43083 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,787 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42648cff[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741846_1029 to 127.0.0.1:43909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:36,798 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:36,956 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/078c231c79ed4a7bb35e249abd3bea04
2024-11-10T02:09:36,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/078c231c79ed4a7bb35e249abd3bea04 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04
2024-11-10T02:09:36,972 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04, entries=1, sequenceid=34, filesize=5.9 K
2024-11-10T02:09:36,974 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f26e09cb29c8f06b526c39c3f659f783 in 448ms, sequenceid=34, compaction requested=true
2024-11-10T02:09:36,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:36,974 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-10T02:09:36,974 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:36,974 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98 because midkey is the same as first or last row
2024-11-10T02:09:36,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f26e09cb29c8f06b526c39c3f659f783:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:09:36,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:36,975 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:09:36,976 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:09:36,977 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1541): f26e09cb29c8f06b526c39c3f659f783/info is initiating minor compaction (all files)
2024-11-10T02:09:36,977 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f26e09cb29c8f06b526c39c3f659f783/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:36,977 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04] into tmpdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp, totalSize=28.2 K
2024-11-10T02:09:36,978 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468f5a42792143cca614cd863bfa281e, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731204570389
2024-11-10T02:09:36,978 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2be0310cd44829a2b0b20d869eaf98, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731204574472
2024-11-10T02:09:36,979 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078c231c79ed4a7bb35e249abd3bea04, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731204576524
2024-11-10T02:09:37,007 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f26e09cb29c8f06b526c39c3f659f783#info#compaction#21 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:09:37,007 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/0e7eadccec054fe1b6055391c6e2dfd8 is 1080, key is row0002/info:/1731204570389/Put/seqid=0
2024-11-10T02:09:37,013 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43083
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,013 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35770 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045 to mirror 127.0.0.1:43083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,013 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:37,013 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045
2024-11-10T02:09:37,014 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35770 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:37,014 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35770 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35770 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,014 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:37,017 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,017 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35780 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,017 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:37,017 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046
2024-11-10T02:09:37,017 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35780 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:37,017 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35780 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35780 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,018 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:37,019 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,020 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:37,020 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741864_1047
2024-11-10T02:09:37,024 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:37,027 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35784 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,027 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:37,027 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048
2024-11-10T02:09:37,027 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35784 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:37,027 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35784 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35784 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,028 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:37,029 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:37,029 WARN  [IPC Server handler 2 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:37,029 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:37,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741866_1049 (size=17994)
2024-11-10T02:09:37,041 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/0e7eadccec054fe1b6055391c6e2dfd8 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8
2024-11-10T02:09:37,053 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f26e09cb29c8f06b526c39c3f659f783/info of f26e09cb29c8f06b526c39c3f659f783 into 0e7eadccec054fe1b6055391c6e2dfd8(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:37,053 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., storeName=f26e09cb29c8f06b526c39c3f659f783/info, priority=13, startTime=1731204576974; duration=0sec
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 because midkey is the same as first or last row
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-10T02:09:37,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 because midkey is the same as first or last row
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 because midkey is the same as first or last row
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:37,054 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f26e09cb29c8f06b526c39c3f659f783:info
2024-11-10T02:09:37,860 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:37,949 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-10T02:09:37,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d8077195c9654baca46c6a8713f1e80a is 1079, key is tmprow/info:/1731204577947/Put/seqid=0
2024-11-10T02:09:37,956 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,956 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35794 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050 to mirror 127.0.0.1:43909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,956 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:37,956 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050
2024-11-10T02:09:37,957 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35794 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050] {}] datanode.BlockReceiver(316): Block 1073741867 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:37,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35794 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741867_1050] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35794 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:37,957 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:37,958 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,959 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:37,959 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741868_1051
2024-11-10T02:09:37,959 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:37,961 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,961 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:37,961 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741869_1052
2024-11-10T02:09:37,962 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:37,963 WARN  [Thread-959 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:37,964 WARN  [Thread-959 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:37,964 WARN  [Thread-959 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741870_1053
2024-11-10T02:09:37,964 WARN  [Thread-959 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:37,965 WARN  [IPC Server handler 0 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:37,965 WARN  [IPC Server handler 0 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:37,966 WARN  [IPC Server handler 0 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:37,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741871_1054 (size=6027)
2024-11-10T02:09:38,235 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,370 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d8077195c9654baca46c6a8713f1e80a
2024-11-10T02:09:38,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d8077195c9654baca46c6a8713f1e80a as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a
2024-11-10T02:09:38,388 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a, entries=1, sequenceid=45, filesize=5.9 K
2024-11-10T02:09:38,389 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f26e09cb29c8f06b526c39c3f659f783 in 441ms, sequenceid=45, compaction requested=false
2024-11-10T02:09:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-10T02:09:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:38,390 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 because midkey is the same as first or last row
2024-11-10T02:09:38,397 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]
2024-11-10T02:09:38,398 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,398 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C41989%2C1731204554095:(num 1731204576369) roll requested
2024-11-10T02:09:38,398 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204578398
2024-11-10T02:09:38,405 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,405 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:38,405 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741872_1055
2024-11-10T02:09:38,406 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:38,417 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43083
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,417 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35816 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056 to mirror 127.0.0.1:43083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:38,418 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:38,418 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056
2024-11-10T02:09:38,418 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35816 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-10T02:09:38,418 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35816 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35816 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:38,419 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:38,421 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,422 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:38,422 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741874_1057
2024-11-10T02:09:38,422 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:38,425 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:38,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35820 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:38,425 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:38,425 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35820 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-10T02:09:38,425 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058
2024-11-10T02:09:38,425 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35820 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35820 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:38,426 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:38,429 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:38,429 WARN  [IPC Server handler 2 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:38,429 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:38,433 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:38,433 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:38,433 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:38,434 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:38,434 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:38,434 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204576369 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204578398
2024-11-10T02:09:38,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741856_1039 (size=13591)
2024-11-10T02:09:38,440 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45991:45991)]
2024-11-10T02:09:38,440 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:38,440 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204576369 is not closed yet, will try archiving it next time
2024-11-10T02:09:38,442 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204572332 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs/0c4d4d227d60%2C41989%2C1731204554095.1731204572332
2024-11-10T02:09:38,838 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 is not closed yet, will try archiving it next time
2024-11-10T02:09:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:39,370 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-10T02:09:39,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d9d17b36269e4525b2e760fd65d2a49a is 1079, key is tmprow/info:/1731204579369/Put/seqid=0
2024-11-10T02:09:39,378 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,378 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:39,379 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741877_1060
2024-11-10T02:09:39,379 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:39,382 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,382 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:39,382 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741878_1061
2024-11-10T02:09:39,382 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:39,384 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,385 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:39,385 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741879_1062
2024-11-10T02:09:39,385 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:39,387 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,387 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:39,387 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741880_1063
2024-11-10T02:09:39,388 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:39,389 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:39,389 WARN  [IPC Server handler 2 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:39,389 WARN  [IPC Server handler 2 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:39,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741881_1064 (size=6027)
2024-11-10T02:09:39,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42648cff[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741841_1024 to 127.0.0.1:43083 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:39,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@edd0c89[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741861_1044 to 127.0.0.1:41599 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:39,795 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d9d17b36269e4525b2e760fd65d2a49a
2024-11-10T02:09:39,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/d9d17b36269e4525b2e760fd65d2a49a as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a
2024-11-10T02:09:39,814 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a, entries=1, sequenceid=55, filesize=5.9 K
2024-11-10T02:09:39,816 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for f26e09cb29c8f06b526c39c3f659f783 in 446ms, sequenceid=55, compaction requested=true
2024-11-10T02:09:39,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-10T02:09:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 because midkey is the same as first or last row
2024-11-10T02:09:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f26e09cb29c8f06b526c39c3f659f783:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:09:39,817 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:09:39,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:39,819 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:09:39,819 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1541): f26e09cb29c8f06b526c39c3f659f783/info is initiating minor compaction (all files)
2024-11-10T02:09:39,819 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f26e09cb29c8f06b526c39c3f659f783/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:39,820 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a] into tmpdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp, totalSize=29.3 K
2024-11-10T02:09:39,820 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e7eadccec054fe1b6055391c6e2dfd8, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731204570389
2024-11-10T02:09:39,821 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8077195c9654baca46c6a8713f1e80a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731204577947
2024-11-10T02:09:39,822 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d17b36269e4525b2e760fd65d2a49a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731204579369
2024-11-10T02:09:39,847 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f26e09cb29c8f06b526c39c3f659f783#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:09:39,848 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/a67fb38673684a2898f66dd1ba73be54 is 1080, key is row0002/info:/1731204570389/Put/seqid=0
2024-11-10T02:09:39,854 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,854 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]) is bad.
2024-11-10T02:09:39,854 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741882_1065
2024-11-10T02:09:39,855 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43083,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]
2024-11-10T02:09:39,857 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,858 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:39,858 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741883_1066
2024-11-10T02:09:39,859 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:39,860 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,861 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,861 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:39,861 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741884_1067
2024-11-10T02:09:39,862 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:39,865 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41599
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:39,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35854 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068 to mirror 127.0.0.1:41599
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:39,866 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:39,866 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068
2024-11-10T02:09:39,866 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35854 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:39,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:35854 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35854 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:39,867 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:39,868 WARN  [IPC Server handler 0 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-10T02:09:39,869 WARN  [IPC Server handler 0 on default port 37393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-10T02:09:39,869 WARN  [IPC Server handler 0 on default port 37393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-10T02:09:39,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741886_1069 (size=18097)
2024-11-10T02:09:39,893 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/a67fb38673684a2898f66dd1ba73be54 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54
2024-11-10T02:09:39,905 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f26e09cb29c8f06b526c39c3f659f783/info of f26e09cb29c8f06b526c39c3f659f783 into a67fb38673684a2898f66dd1ba73be54(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:09:39,905 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:39,905 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., storeName=f26e09cb29c8f06b526c39c3f659f783/info, priority=13, startTime=1731204579817; duration=0sec
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 because midkey is the same as first or last row
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 because midkey is the same as first or last row
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 because midkey is the same as first or last row
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:39,906 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f26e09cb29c8f06b526c39c3f659f783:info
2024-11-10T02:09:40,235 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:40,441 WARN  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-10T02:09:40,441 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:40,598 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:09:40,603 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:09:40,604 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:09:40,604 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:09:40,604 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:09:40,605 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@441dcfc4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:09:40,605 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@381443d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:09:40,742 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14b00457{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/java.io.tmpdir/jetty-localhost-43421-hadoop-hdfs-3_4_1-tests_jar-_-any-120703808282615124/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:09:40,743 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22c6c03b{HTTP/1.1, (http/1.1)}{localhost:43421}
2024-11-10T02:09:40,743 INFO  [Time-limited test {}] server.Server(415): Started @138219ms
2024-11-10T02:09:40,745 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:09:40,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@edd0c89[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741866_1049 to 127.0.0.1:43083 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:40,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42648cff[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741871_1054 to 127.0.0.1:43909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:41,392 WARN  [Thread-993 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:09:41,402 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x261b79fd32c55459 with lease ID 0x9fd12e564a2250d6: from storage DS-8220fa32-3509-4176-a4fd-14808e07cb0a node DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:09:41,402 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x261b79fd32c55459 with lease ID 0x9fd12e564a2250d6: from storage DS-f7367f00-b920-4db9-b524-082b64362b51 node DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:09:41,861 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:42,236 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:42,441 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:42,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@edd0c89[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741856_1039 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:42,788 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42648cff[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741881_1064 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:43,787 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42648cff[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37331, datanodeUuid=0764c3e9-36d2-42ec-8be4-73d65693d3b8, infoPort=45991, infoSecurePort=0, ipcPort=40349, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741886_1069 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:43,861 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:43,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:09:44,236 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,442 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,558 ERROR [FSHLog-0-hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData-prefix:0c4d4d227d60,38021,1731204553917 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,558 WARN  [FSHLog-0-hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData-prefix:0c4d4d227d60,38021,1731204553917 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,559 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C38021%2C1731204553917:(num 1731204554269) roll requested
2024-11-10T02:09:44,559 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C38021%2C1731204553917.1731204584559
2024-11-10T02:09:44,562 WARN  [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,563 WARN  [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK], DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]) is bad.
2024-11-10T02:09:44,563 WARN  [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741887_1070
2024-11-10T02:09:44,564 WARN  [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]
2024-11-10T02:09:44,565 WARN  [Thread-1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,566 WARN  [Thread-1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK], DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]) is bad.
2024-11-10T02:09:44,566 WARN  [Thread-1013 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741888_1071
2024-11-10T02:09:44,566 WARN  [Thread-1013 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43909,DS-b3ec7d89-108a-4f7e-91c3-1ab4f16f94ca,DISK]
2024-11-10T02:09:44,572 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:44,573 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:44,573 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:44,573 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:44,573 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:44,573 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204584559
2024-11-10T02:09:44,574 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,574 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:44,574 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269
2024-11-10T02:09:44,574 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:45991:45991)]
2024-11-10T02:09:44,574 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 is not closed yet, will try archiving it next time
2024-11-10T02:09:44,575 WARN  [IPC Server handler 0 on default port 37393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-11-10T02:09:44,575 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 after 1ms
2024-11-10T02:09:45,862 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:46,442 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:47,862 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:48,442 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:48,577 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 after 4002ms
2024-11-10T02:09:49,862 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:50,443 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:51,421 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@106faeb2 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:41599,null,null])
java.net.ConnectException: Call From 0c4d4d227d60/172.17.0.3 to localhost:33647 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-10T02:09:51,863 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:52,359 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204554915 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs/0c4d4d227d60%2C41989%2C1731204554095.1731204554915
2024-11-10T02:09:52,360 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204576369 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs/0c4d4d227d60%2C41989%2C1731204554095.1731204576369
2024-11-10T02:09:52,443 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:53,693 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.1731204593693
2024-11-10T02:09:53,703 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:53,703 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:53,703 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:53,703 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:53,703 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:53,703 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204578398 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.1731204593693
2024-11-10T02:09:53,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741876_1059 (size=12911)
2024-11-10T02:09:53,708 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45991:45991),(127.0.0.1/127.0.0.1:36781:36781)]
2024-11-10T02:09:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:53,714 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-10T02:09:53,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/2203e4721aba42c982f77f51742634a5 is 1080, key is row0013/info:/1731204593710/Put/seqid=0
2024-11-10T02:09:53,735 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741891_1075 (size=8190)
2024-11-10T02:09:53,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741891_1075 (size=8190)
2024-11-10T02:09:53,739 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/2203e4721aba42c982f77f51742634a5
2024-11-10T02:09:53,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/2203e4721aba42c982f77f51742634a5 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5
2024-11-10T02:09:53,763 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5, entries=3, sequenceid=66, filesize=8.0 K
2024-11-10T02:09:53,764 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for f26e09cb29c8f06b526c39c3f659f783 in 50ms, sequenceid=66, compaction requested=false
2024-11-10T02:09:53,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:53,764 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-10T02:09:53,764 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:53,764 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 because midkey is the same as first or last row
2024-11-10T02:09:53,863 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41989 {}] regionserver.HRegion(8855): Flush requested on f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:53,950 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-10T02:09:53,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5ba16585c5c348a0b1e57acbfb4586a1 is 1080, key is row0015/info:/1731204593716/Put/seqid=0
2024-11-10T02:09:53,959 WARN  [Thread-1036 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:53,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43442 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4]'}, localName='127.0.0.1:40739', datanodeUuid='29439ccc-285b-473b-a786-ec1b9ce6055d', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:53,959 WARN  [Thread-1036 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:53,959 WARN  [Thread-1036 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076
2024-11-10T02:09:53,959 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43442 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:53,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43442 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741892_1076] {}] datanode.DataXceiver(331): 127.0.0.1:40739:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43442 dst: /127.0.0.1:40739
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:53,960 WARN  [Thread-1036 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:53,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741893_1077 (size=14660)
2024-11-10T02:09:53,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741893_1077 (size=14660)
2024-11-10T02:09:53,973 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5ba16585c5c348a0b1e57acbfb4586a1
2024-11-10T02:09:53,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/5ba16585c5c348a0b1e57acbfb4586a1 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1
2024-11-10T02:09:53,998 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1, entries=9, sequenceid=79, filesize=14.3 K
2024-11-10T02:09:53,999 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for f26e09cb29c8f06b526c39c3f659f783 in 50ms, sequenceid=79, compaction requested=true
2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 because midkey is the same as first or last row
2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f26e09cb29c8f06b526c39c3f659f783:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:09:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:54,000 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:09:54,002 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:09:54,002 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1541): f26e09cb29c8f06b526c39c3f659f783/info is initiating minor compaction (all files)
2024-11-10T02:09:54,003 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f26e09cb29c8f06b526c39c3f659f783/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,003 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1] into tmpdir=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp, totalSize=40.0 K
2024-11-10T02:09:54,003 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting a67fb38673684a2898f66dd1ba73be54, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731204570389
2024-11-10T02:09:54,004 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2203e4721aba42c982f77f51742634a5, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731204580386
2024-11-10T02:09:54,004 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba16585c5c348a0b1e57acbfb4586a1, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731204593716
2024-11-10T02:09:54,020 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f26e09cb29c8f06b526c39c3f659f783#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:09:54,021 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/de1b63d8afd14a4b9874286add37e298 is 1080, key is row0002/info:/1731204570389/Put/seqid=0
2024-11-10T02:09:54,023 WARN  [Thread-1044 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,023 WARN  [Thread-1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,023 WARN  [Thread-1044 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741894_1078
2024-11-10T02:09:54,024 WARN  [Thread-1044 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741895_1079 (size=28989)
2024-11-10T02:09:54,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741895_1079 (size=28989)
2024-11-10T02:09:54,053 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/de1b63d8afd14a4b9874286add37e298 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/de1b63d8afd14a4b9874286add37e298
2024-11-10T02:09:54,070 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f26e09cb29c8f06b526c39c3f659f783/info of f26e09cb29c8f06b526c39c3f659f783 into de1b63d8afd14a4b9874286add37e298(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:09:54,070 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f26e09cb29c8f06b526c39c3f659f783:

2024-11-10T02:09:54,070 INFO  [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., storeName=f26e09cb29c8f06b526c39c3f659f783/info, priority=13, startTime=1731204594000; duration=0sec
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/de1b63d8afd14a4b9874286add37e298 because midkey is the same as first or last row
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/de1b63d8afd14a4b9874286add37e298 because midkey is the same as first or last row
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/de1b63d8afd14a4b9874286add37e298 because midkey is the same as first or last row
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:09:54,071 DEBUG [RS:0;0c4d4d227d60:41989-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f26e09cb29c8f06b526c39c3f659f783:info
2024-11-10T02:09:54,397 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@21b53e68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741833_1019 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,397 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1db766c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741835_1011 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,444 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-10T02:09:54,444 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,557 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:09:54,557 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:54,557 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:54,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:54,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:54,558 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:09:54,558 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:09:54,558 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1609523067, stopped=false
2024-11-10T02:09:54,558 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,38021,1731204553917
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:09:54,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:09:54,702 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:09:54,703 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:09:54,703 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:54,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:54,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:54,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:54,704 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,41989,1731204554095' *****
2024-11-10T02:09:54,704 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:09:54,704 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,46049,1731204555690' *****
2024-11-10T02:09:54,704 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:09:54,704 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:09:54,704 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:09:54,704 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:09:54,704 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:09:54,704 INFO  [RS:1;0c4d4d227d60:46049 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:09:54,704 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:09:54,704 INFO  [RS:0;0c4d4d227d60:41989 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:09:54,705 INFO  [RS:0;0c4d4d227d60:41989 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:09:54,705 INFO  [RS:1;0c4d4d227d60:46049 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:09:54,705 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:54,705 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:54,705 INFO  [RS:1;0c4d4d227d60:46049 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0c4d4d227d60:46049.
2024-11-10T02:09:54,705 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(3091): Received CLOSE for f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:54,705 DEBUG [RS:1;0c4d4d227d60:46049 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:54,705 DEBUG [RS:1;0c4d4d227d60:46049 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:54,705 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,46049,1731204555690; all regions closed.
2024-11-10T02:09:54,705 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,41989,1731204554095
2024-11-10T02:09:54,705 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:09:54,705 INFO  [RS:0;0c4d4d227d60:41989 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:41989.
2024-11-10T02:09:54,705 DEBUG [RS:0;0c4d4d227d60:41989 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:09:54,706 DEBUG [RS:0;0c4d4d227d60:41989 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:54,706 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,706 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:09:54,706 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:09:54,706 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:09:54,706 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f26e09cb29c8f06b526c39c3f659f783, disabling compactions & flushes
2024-11-10T02:09:54,706 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,706 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:09:54,706 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,706 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783. after waiting 0 ms
2024-11-10T02:09:54,706 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,706 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f26e09cb29c8f06b526c39c3f659f783 1/1 column families, dataSize=2.10 KB heapSize=2.50 KB
2024-11-10T02:09:54,707 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,708 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-10T02:09:54,708 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f26e09cb29c8f06b526c39c3f659f783=TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.}
2024-11-10T02:09:54,708 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f26e09cb29c8f06b526c39c3f659f783
2024-11-10T02:09:54,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:09:54,709 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:09:54,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:09:54,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:09:54,709 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:09:54,709 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-10T02:09:54,709 ERROR [FSHLog-0-hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7-prefix:0c4d4d227d60,41989,1731204554095.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,709 WARN  [FSHLog-0-hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7-prefix:0c4d4d227d60,41989,1731204554095.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,710 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C41989%2C1731204554095.meta:.meta(num 1731204555357) roll requested
2024-11-10T02:09:54,710 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,710 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,710 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,710 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C41989%2C1731204554095.meta.1731204594710.meta
2024-11-10T02:09:54,711 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,711 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,711 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
2024-11-10T02:09:54,712 WARN  [IPC Server handler 0 on default port 37393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741837_1013
2024-11-10T02:09:54,712 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 after 1ms
2024-11-10T02:09:54,714 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,714 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741896_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,714 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741896_1081
2024-11-10T02:09:54,715 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/e1424f56a480451dae4049f969d949f4 is 1079, key is tmprow/info:/1731204594355/Put/seqid=0
2024-11-10T02:09:54,715 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,718 WARN  [Thread-1054 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1083
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,718 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:54148 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8]'}, localName='127.0.0.1:37331', datanodeUuid='0764c3e9-36d2-42ec-8be4-73d65693d3b8', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,719 WARN  [Thread-1054 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37331,DS-f02eab7e-6719-4f8c-a88b-f9fa67ddf567,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,719 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:54148 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083] {}] datanode.BlockReceiver(316): Block 1073741898 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:54,719 WARN  [Thread-1054 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083
2024-11-10T02:09:54,719 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:54148 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741898_1083] {}] datanode.DataXceiver(331): 127.0.0.1:37331:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54148 dst: /127.0.0.1:37331
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,720 WARN  [Thread-1054 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741899_1084 (size=6027)
2024-11-10T02:09:54,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741899_1084 (size=6027)
2024-11-10T02:09:54,732 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.10 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/e1424f56a480451dae4049f969d949f4
2024-11-10T02:09:54,732 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,732 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,733 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,733 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,733 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:54,733 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204594710.meta
2024-11-10T02:09:54,734 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,734 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41599,DS-ce0bcbb4-164c-4e4c-8a90-00cbc9af5f3c,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,734 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
2024-11-10T02:09:54,735 WARN  [IPC Server handler 0 on default port 37393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta has not been closed. Lease recovery is in progress. RecoveryId = 1085 for block blk_1073741834_1010
2024-11-10T02:09:54,735 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36781:36781),(127.0.0.1/127.0.0.1:45991:45991)]
2024-11-10T02:09:54,735 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta is not closed yet, will try archiving it next time
2024-11-10T02:09:54,735 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta after 1ms
2024-11-10T02:09:54,747 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/.tmp/info/e1424f56a480451dae4049f969d949f4 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/e1424f56a480451dae4049f969d949f4
2024-11-10T02:09:54,761 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/e1424f56a480451dae4049f969d949f4, entries=1, sequenceid=85, filesize=5.9 K
2024-11-10T02:09:54,764 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:54,765 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/info/d737156c8bc340c5b41fae4cd9b9c44a is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783./info:regioninfo/1731204556278/Put/seqid=0
2024-11-10T02:09:54,767 WARN  [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,768 WARN  [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK], DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,768 WARN  [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741900_1086
2024-11-10T02:09:54,768 WARN  [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,769 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.10 KB/2150, heapSize ~2.48 KB/2544, currentSize=0 B/0 for f26e09cb29c8f06b526c39c3f659f783 in 63ms, sequenceid=85, compaction requested=false
2024-11-10T02:09:54,776 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1] to archive
2024-11-10T02:09:54,778 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:09:54,783 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/468f5a42792143cca614cd863bfa281e
2024-11-10T02:09:54,785 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5c2be0310cd44829a2b0b20d869eaf98
2024-11-10T02:09:54,787 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/0e7eadccec054fe1b6055391c6e2dfd8
2024-11-10T02:09:54,792 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/078c231c79ed4a7bb35e249abd3bea04
2024-11-10T02:09:54,794 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d8077195c9654baca46c6a8713f1e80a
2024-11-10T02:09:54,796 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/a67fb38673684a2898f66dd1ba73be54
2024-11-10T02:09:54,798 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/d9d17b36269e4525b2e760fd65d2a49a
2024-11-10T02:09:54,800 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/2203e4721aba42c982f77f51742634a5
2024-11-10T02:09:54,805 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/info/5ba16585c5c348a0b1e57acbfb4586a1
2024-11-10T02:09:54,806 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0c4d4d227d60:38021 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-10T02:09:54,807 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [468f5a42792143cca614cd863bfa281e=10347, 5c2be0310cd44829a2b0b20d869eaf98=12506, 0e7eadccec054fe1b6055391c6e2dfd8=17994, 078c231c79ed4a7bb35e249abd3bea04=6027, d8077195c9654baca46c6a8713f1e80a=6027, a67fb38673684a2898f66dd1ba73be54=18097, d9d17b36269e4525b2e760fd65d2a49a=6027, 2203e4721aba42c982f77f51742634a5=8190, 5ba16585c5c348a0b1e57acbfb4586a1=14660]
2024-11-10T02:09:54,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741901_1087 (size=7089)
2024-11-10T02:09:54,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741901_1087 (size=7089)
2024-11-10T02:09:54,812 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/info/d737156c8bc340c5b41fae4cd9b9c44a
2024-11-10T02:09:54,819 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:09:54,820 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:09:54,831 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/f26e09cb29c8f06b526c39c3f659f783/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=1
2024-11-10T02:09:54,832 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,832 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f26e09cb29c8f06b526c39c3f659f783:
Waiting for close lock at 1731204594706Running coprocessor pre-close hooks at 1731204594706Disabling compacts and flushes for region at 1731204594706Disabling writes for close at 1731204594706Obtaining lock to block concurrent updates at 1731204594706Preparing flush snapshotting stores in f26e09cb29c8f06b526c39c3f659f783 at 1731204594706Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783., syncing WAL and waiting on mvcc, flushsize=dataSize=2150, getHeapSize=2544, getOffHeapSize=0, getCellsCount=2 at 1731204594707 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783. at 1731204594708 (+1 ms)Flushing f26e09cb29c8f06b526c39c3f659f783/info: creating writer at 1731204594708Flushing f26e09cb29c8f06b526c39c3f659f783/info: appending metadata at 1731204594714 (+6 ms)Flushing f26e09cb29c8f06b526c39c3f659f783/info: closing flushed file at 1731204594714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d04eb28: reopening flushed file at 1731204594745 (+31 ms)Finished flush of dataSize ~2.10 KB/2150, heapSize ~2.48 KB/2544, currentSize=0 B/0 for f26e09cb29c8f06b526c39c3f659f783 in 63ms, sequenceid=85, compaction requested=false at 1731204594769 (+24 ms)Writing region close event to WAL at 1731204594819 (+50 ms)Running coprocessor post-close hooks at 1731204594832 (+13 ms)Closed at 1731204594832
2024-11-10T02:09:54,832 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731204555899.f26e09cb29c8f06b526c39c3f659f783.
2024-11-10T02:09:54,846 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/ns/97f8bfb2116f435fb64d5a3d0091b30a is 43, key is default/ns:d/1731204555535/Put/seqid=0
2024-11-10T02:09:54,850 WARN  [Thread-1072 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1088
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43504 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4]'}, localName='127.0.0.1:40739', datanodeUuid='29439ccc-285b-473b-a786-ec1b9ce6055d', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,850 WARN  [Thread-1072 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,850 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43504 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088] {}] datanode.BlockReceiver(316): Block 1073741902 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:54,850 WARN  [Thread-1072 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088
2024-11-10T02:09:54,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43504 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741902_1088] {}] datanode.DataXceiver(331): 127.0.0.1:40739:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43504 dst: /127.0.0.1:40739
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,851 WARN  [Thread-1072 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,860 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741903_1089 (size=5153)
2024-11-10T02:09:54,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741903_1089 (size=5153)
2024-11-10T02:09:54,862 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/ns/97f8bfb2116f435fb64d5a3d0091b30a
2024-11-10T02:09:54,897 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/table/328d6b0540fd4520abe1e1d2613d4312 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731204556301/Put/seqid=0
2024-11-10T02:09:54,899 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:09:54,899 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:09:54,901 WARN  [Thread-1080 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1090
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38203
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:09:54,901 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43518 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4]'}, localName='127.0.0.1:40739', datanodeUuid='29439ccc-285b-473b-a786-ec1b9ce6055d', xmitsInProgress=0}:Exception transferring block BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090 to mirror 127.0.0.1:38203
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,901 WARN  [Thread-1080 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40739,DS-8220fa32-3509-4176-a4fd-14808e07cb0a,DISK], DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]) is bad.
2024-11-10T02:09:54,901 WARN  [Thread-1080 {}] hdfs.DataStreamer(1850): Abandoning BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090
2024-11-10T02:09:54,901 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43518 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090] {}] datanode.BlockReceiver(316): Block 1073741904 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-10T02:09:54,901 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1302774573_22 at /127.0.0.1:43518 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741904_1090] {}] datanode.DataXceiver(331): 127.0.0.1:40739:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43518 dst: /127.0.0.1:40739
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:54,902 WARN  [Thread-1080 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38203,DS-84e5a623-994d-4ed2-af1a-8e38ea4f19f8,DISK]
2024-11-10T02:09:54,908 DEBUG [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-10T02:09:54,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741905_1091 (size=5424)
2024-11-10T02:09:54,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741905_1091 (size=5424)
2024-11-10T02:09:54,915 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/table/328d6b0540fd4520abe1e1d2613d4312
2024-11-10T02:09:54,929 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/info/d737156c8bc340c5b41fae4cd9b9c44a as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/info/d737156c8bc340c5b41fae4cd9b9c44a
2024-11-10T02:09:54,938 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/info/d737156c8bc340c5b41fae4cd9b9c44a, entries=10, sequenceid=11, filesize=6.9 K
2024-11-10T02:09:54,940 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/ns/97f8bfb2116f435fb64d5a3d0091b30a as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/ns/97f8bfb2116f435fb64d5a3d0091b30a
2024-11-10T02:09:54,952 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/ns/97f8bfb2116f435fb64d5a3d0091b30a, entries=2, sequenceid=11, filesize=5.0 K
2024-11-10T02:09:54,954 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/.tmp/table/328d6b0540fd4520abe1e1d2613d4312 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/table/328d6b0540fd4520abe1e1d2613d4312
2024-11-10T02:09:54,963 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/table/328d6b0540fd4520abe1e1d2613d4312, entries=2, sequenceid=11, filesize=5.3 K
2024-11-10T02:09:54,964 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 255ms, sequenceid=11, compaction requested=false
2024-11-10T02:09:54,971 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-10T02:09:54,972 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:54,972 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:54,972 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204594708Running coprocessor pre-close hooks at 1731204594708Disabling compacts and flushes for region at 1731204594708Disabling writes for close at 1731204594709 (+1 ms)Obtaining lock to block concurrent updates at 1731204594709Preparing flush snapshotting stores in 1588230740 at 1731204594709Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731204594709Flushing stores of hbase:meta,,1.1588230740 at 1731204594736 (+27 ms)Flushing 1588230740/info: creating writer at 1731204594737 (+1 ms)Flushing 1588230740/info: appending metadata at 1731204594765 (+28 ms)Flushing 1588230740/info: closing flushed file at 1731204594765Flushing 1588230740/ns: creating writer at 1731204594823 (+58 ms)Flushing 1588230740/ns: appending metadata at 1731204594845 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731204594846 (+1 ms)Flushing 1588230740/table: creating writer at 1731204594871 (+25 ms)Flushing 1588230740/table: appending metadata at 1731204594897 (+26 ms)Flushing 1588230740/table: closing flushed file at 1731204594897Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22181f2f: reopening flushed file at 1731204594923 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c70a141: reopening flushed file at 1731204594939 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7340b787: reopening flushed file at 1731204594953 (+14 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 255ms, sequenceid=11, compaction requested=false at 1731204594965 (+12 ms)Writing region close event to WAL at 1731204594966 (+1 ms)Running coprocessor post-close hooks at 1731204594972 (+6 ms)Closed at 1731204594972
2024-11-10T02:09:54,972 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:09:55,109 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,41989,1731204554095; all regions closed.
2024-11-10T02:09:55,109 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:55,109 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:55,110 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:55,110 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:55,110 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:09:55,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741897_1082 (size=825)
2024-11-10T02:09:55,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741897_1082 (size=825)
2024-11-10T02:09:55,862 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:55,931 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-10T02:09:55,931 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-10T02:09:56,397 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1db766c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741831_1007 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:56,397 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@21b53e68[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:40739, datanodeUuid=29439ccc-285b-473b-a786-ec1b9ce6055d, infoPort=36781, infoSecurePort=0, ipcPort=35177, storageInfo=lv=-57;cid=testClusterID;nsid=638160718;c=1731204551379):Failed to transfer BP-1233850745-172.17.0.3-1731204551379:blk_1073741829_1005 to 127.0.0.1:38203 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:09:57,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741836_1012 (size=76)
2024-11-10T02:09:57,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:09:58,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:09:58,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:09:58,714 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 after 4002ms
2024-11-10T02:09:58,737 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta after 4002ms
2024-11-10T02:09:59,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-10T02:09:59,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:09:59,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:09:59,711 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-10T02:09:59,713 DEBUG [RS:1;0c4d4d227d60:46049 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs
2024-11-10T02:09:59,713 INFO  [RS:1;0c4d4d227d60:46049 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C46049%2C1731204555690:(num 1731204555996)
2024-11-10T02:09:59,713 DEBUG [RS:1;0c4d4d227d60:46049 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:09:59,713 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:09:59,714 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:09:59,714 INFO  [RS:1;0c4d4d227d60:46049 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46049
2024-11-10T02:09:59,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:09:59,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:09:59,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,46049,1731204555690
2024-11-10T02:09:59,739 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,46049,1731204555690]
2024-11-10T02:09:59,740 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:09:59,752 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,46049,1731204555690 already deleted, retry=false
2024-11-10T02:09:59,752 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,46049,1731204555690 expired; onlineServers=1
2024-11-10T02:09:59,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,854 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:59,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46049-0x1012281184e0002, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:09:59,863 INFO  [RS:1;0c4d4d227d60:46049 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:09:59,863 INFO  [RS:1;0c4d4d227d60:46049 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,46049,1731204555690; zookeeper connection closed.
2024-11-10T02:09:59,863 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e988cfa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e988cfa
2024-11-10T02:09:59,880 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:09:59,881 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,110 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-10T02:10:00,114 DEBUG [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs
2024-11-10T02:10:00,114 INFO  [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C41989%2C1731204554095.meta:.meta(num 1731204594710)
2024-11-10T02:10:00,115 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:00,115 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:00,115 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:00,115 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:00,115 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:00,383 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:10:00,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,403 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,403 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,410 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,413 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:00,433 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1040821705_22 at /127.0.0.1:43404 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741890_1074] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 317ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4/, blockId=1073741890, seqno=23
2024-11-10T02:10:00,433 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1040821705_22 at /127.0.0.1:54092 [Receiving block BP-1233850745-172.17.0.3-1731204551379:blk_1073741890_1074] {}] datanode.BlockReceiver(464): Slow flushOrSync took 317ms (threshold=300ms), isSync:false, flushTotalNanos=317354651ns, volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7/, blockId=1073741890, seqno=23
2024-11-10T02:10:00,435 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741890_1074 (size=19324)
2024-11-10T02:10:00,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741890_1074 (size=19324)
2024-11-10T02:10:00,517 DEBUG [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs
2024-11-10T02:10:00,517 INFO  [RS:0;0c4d4d227d60:41989 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C41989%2C1731204554095:(num 1731204593693)
2024-11-10T02:10:00,518 DEBUG [RS:0;0c4d4d227d60:41989 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:00,518 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:10:00,518 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:10:00,518 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-10T02:10:00,518 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:10:00,518 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:10:00,518 INFO  [RS:0;0c4d4d227d60:41989 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41989
2024-11-10T02:10:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:10:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,41989,1731204554095
2024-11-10T02:10:00,628 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:10:00,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:00,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:00,741 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,41989,1731204554095]
2024-11-10T02:10:00,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741876_1059 (size=12911)
2024-11-10T02:10:00,841 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,41989,1731204554095 already deleted, retry=false
2024-11-10T02:10:00,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:00,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41989-0x1012281184e0001, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:00,841 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,41989,1731204554095 expired; onlineServers=0
2024-11-10T02:10:00,841 INFO  [RS:0;0c4d4d227d60:41989 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:10:00,841 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,38021,1731204553917' *****
2024-11-10T02:10:00,841 INFO  [RS:0;0c4d4d227d60:41989 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,41989,1731204554095; zookeeper connection closed.
2024-11-10T02:10:00,841 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:10:00,841 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:10:00,841 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:10:00,841 DEBUG [M:0;0c4d4d227d60:38021 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:10:00,842 DEBUG [M:0;0c4d4d227d60:38021 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:10:00,842 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ee4eff7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ee4eff7
2024-11-10T02:10:00,842 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-10T02:10:00,842 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:10:00,842 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:10:00,842 DEBUG [M:0;0c4d4d227d60:38021 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:10:00,842 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204554559 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204554559,5,FailOnTimeoutGroup]
2024-11-10T02:10:00,842 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:10:00,842 INFO  [M:0;0c4d4d227d60:38021 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:10:00,842 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204554559 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204554559,5,FailOnTimeoutGroup]
2024-11-10T02:10:00,842 INFO  [M:0;0c4d4d227d60:38021 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:10:00,842 INFO  [M:0;0c4d4d227d60:38021 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:10:00,842 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:10:00,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:10:00,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:00,956 DEBUG [M:0;0c4d4d227d60:38021 {}] zookeeper.ZKUtil(347): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:10:00,956 WARN  [M:0;0c4d4d227d60:38021 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:10:00,957 INFO  [M:0;0c4d4d227d60:38021 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/.lastflushedseqids
2024-11-10T02:10:00,975 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741906_1092 (size=130)
2024-11-10T02:10:00,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741906_1092 (size=130)
2024-11-10T02:10:00,976 INFO  [M:0;0c4d4d227d60:38021 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:10:00,976 INFO  [M:0;0c4d4d227d60:38021 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:10:00,976 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:10:00,977 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:00,977 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:00,977 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:10:00,977 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:00,977 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-10T02:10:00,994 DEBUG [M:0;0c4d4d227d60:38021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dbe8e87005804862b5ceef6278e9585b is 82, key is hbase:meta,,1/info:regioninfo/1731204555400/Put/seqid=0
2024-11-10T02:10:01,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741907_1093 (size=5672)
2024-11-10T02:10:01,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741907_1093 (size=5672)
2024-11-10T02:10:01,001 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dbe8e87005804862b5ceef6278e9585b
2024-11-10T02:10:01,023 DEBUG [M:0;0c4d4d227d60:38021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0e830a29f1a40a5af41759377435daa is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731204556308/Put/seqid=0
2024-11-10T02:10:01,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741908_1094 (size=6256)
2024-11-10T02:10:01,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741908_1094 (size=6256)
2024-11-10T02:10:01,033 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0e830a29f1a40a5af41759377435daa
2024-11-10T02:10:01,039 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f0e830a29f1a40a5af41759377435daa
2024-11-10T02:10:01,054 DEBUG [M:0;0c4d4d227d60:38021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb45ee4b437d481387d3b26e91565aba is 69, key is 0c4d4d227d60,41989,1731204554095/rs:state/1731204554710/Put/seqid=0
2024-11-10T02:10:01,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741909_1095 (size=5224)
2024-11-10T02:10:01,060 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741909_1095 (size=5224)
2024-11-10T02:10:01,060 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb45ee4b437d481387d3b26e91565aba
2024-11-10T02:10:01,085 DEBUG [M:0;0c4d4d227d60:38021 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f4e51c4500e94d0ca7bf88c8b6b5f357 is 52, key is load_balancer_on/state:d/1731204555661/Put/seqid=0
2024-11-10T02:10:01,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741910_1096 (size=5056)
2024-11-10T02:10:01,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741910_1096 (size=5056)
2024-11-10T02:10:01,091 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f4e51c4500e94d0ca7bf88c8b6b5f357
2024-11-10T02:10:01,097 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dbe8e87005804862b5ceef6278e9585b as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dbe8e87005804862b5ceef6278e9585b
2024-11-10T02:10:01,103 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dbe8e87005804862b5ceef6278e9585b, entries=8, sequenceid=60, filesize=5.5 K
2024-11-10T02:10:01,104 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0e830a29f1a40a5af41759377435daa as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0e830a29f1a40a5af41759377435daa
2024-11-10T02:10:01,109 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f0e830a29f1a40a5af41759377435daa
2024-11-10T02:10:01,109 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0e830a29f1a40a5af41759377435daa, entries=6, sequenceid=60, filesize=6.1 K
2024-11-10T02:10:01,112 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eb45ee4b437d481387d3b26e91565aba as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb45ee4b437d481387d3b26e91565aba
2024-11-10T02:10:01,119 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eb45ee4b437d481387d3b26e91565aba, entries=2, sequenceid=60, filesize=5.1 K
2024-11-10T02:10:01,120 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f4e51c4500e94d0ca7bf88c8b6b5f357 as hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f4e51c4500e94d0ca7bf88c8b6b5f357
2024-11-10T02:10:01,127 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f4e51c4500e94d0ca7bf88c8b6b5f357, entries=1, sequenceid=60, filesize=4.9 K
2024-11-10T02:10:01,129 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false
2024-11-10T02:10:01,136 INFO  [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:01,136 DEBUG [M:0;0c4d4d227d60:38021 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204600976Disabling compacts and flushes for region at 1731204600976Disabling writes for close at 1731204600977 (+1 ms)Obtaining lock to block concurrent updates at 1731204600977Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204600977Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1731204600977Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204600978 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204600978Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204600993 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204600993Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204601009 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204601023 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204601023Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204601039 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204601053 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204601053Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204601066 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204601085 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204601085Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68828c01: reopening flushed file at 1731204601096 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24cf4ea8: reopening flushed file at 1731204601103 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76f39b2b: reopening flushed file at 1731204601110 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1580e6c6: reopening flushed file at 1731204601119 (+9 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=60, compaction requested=false at 1731204601129 (+10 ms)Writing region close event to WAL at 1731204601136 (+7 ms)Closed at 1731204601136
2024-11-10T02:10:01,137 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:01,137 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:01,137 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:01,137 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:01,137 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:01,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37331 is added to blk_1073741889_1072 (size=1045)
2024-11-10T02:10:01,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40739 is added to blk_1073741889_1072 (size=1045)
2024-11-10T02:10:01,426 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6587a8bd {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:41599,null,null])
java.net.ConnectException: Call From 0c4d4d227d60/172.17.0.3 to localhost:33647 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-10T02:10:01,586 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/WALs/0c4d4d227d60,38021,1731204553917/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/oldWALs/0c4d4d227d60%2C38021%2C1731204553917.1731204554269
2024-11-10T02:10:01,590 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/MasterData/oldWALs/0c4d4d227d60%2C38021%2C1731204553917.1731204554269 to hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/oldWALs/0c4d4d227d60%2C38021%2C1731204553917.1731204554269$masterlocalwal$
2024-11-10T02:10:01,590 INFO  [M:0;0c4d4d227d60:38021 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:10:01,590 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:10:01,590 INFO  [M:0;0c4d4d227d60:38021 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38021
2024-11-10T02:10:01,591 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:10:01,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:01,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:01,776 INFO  [M:0;0c4d4d227d60:38021 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:10:01,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:01,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38021-0x1012281184e0000, quorum=127.0.0.1:63612, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:01,812 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14b00457{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:01,812 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22c6c03b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:01,812 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:01,813 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@381443d3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:01,813 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@441dcfc4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:01,814 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:01,814 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@28dadd08 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:41599,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:33647 , LocalHost:localPort 0c4d4d227d60/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-10T02:10:01,814 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:01,814 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:01,814 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 29439ccc-285b-473b-a786-ec1b9ce6055d) service to localhost/127.0.0.1:37393
2024-11-10T02:10:01,815 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data3/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:01,816 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data4/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:01,816 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@28dadd08 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1233850745-172.17.0.3-1731204551379:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41599,null,null], DatanodeInfoWithStorage[127.0.0.1:40739,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: Block pool BP-1233850745-172.17.0.3-1731204551379 has not recognized an active NN
2024-11-10T02:10:01,816 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:01,816 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@28dadd08 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40739,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1233850745-172.17.0.3-1731204551379
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:01,816 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@28dadd08 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41599,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1233850745-172.17.0.3-1731204551379
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:01,816 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@28dadd08 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:40739,null,null], DatanodeInfoWithStorage[127.0.0.1:41599,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1233850745-172.17.0.3-1731204551379:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:40739,null,null], DatanodeInfoWithStorage[127.0.0.1:41599,null,null]]
2024-11-10T02:10:01,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d82aaea{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:01,818 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3904e150{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:01,818 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:01,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64b89ed5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:01,818 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3f85c2b2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:01,820 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:01,820 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:01,820 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:01,820 WARN  [BP-1233850745-172.17.0.3-1731204551379 heartbeating to localhost/127.0.0.1:37393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1233850745-172.17.0.3-1731204551379 (Datanode Uuid 0764c3e9-36d2-42ec-8be4-73d65693d3b8) service to localhost/127.0.0.1:37393
2024-11-10T02:10:01,821 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data7/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:01,821 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/cluster_d9af0ad2-1fe5-cd81-8de3-8b60aad3240b/data/data8/current/BP-1233850745-172.17.0.3-1731204551379 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:01,821 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:01,828 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2295376c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:10:01,828 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:01,828 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:01,829 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:01,829 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:01,842 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:10:01,886 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:10:01,895 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 80)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37393 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44293
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37393 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44293
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:37393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37393 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff4f0bf5378.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007ff4f0bf5378.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37393 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=358 (was 217) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4065 (was 5863)
2024-11-10T02:10:01,903 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=358, ProcessCount=11, AvailableMemoryMB=4065
2024-11-10T02:10:01,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:10:01,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.log.dir so I do NOT create it in target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118
2024-11-10T02:10:01,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5b96b5ac-d89e-cb42-e022-44d245d174df/hadoop.tmp.dir so I do NOT create it in target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118
2024-11-10T02:10:01,904 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830, deleteOnExit=true
2024-11-10T02:10:01,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/test.cache.data in system properties and HBase conf
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:10:01,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:10:01,905 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:10:01,906 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:10:01,907 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:10:01,922 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:10:02,423 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:02,428 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:02,429 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:02,430 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:02,430 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:02,431 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:02,432 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40c321ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:02,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37ba1ac4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:02,541 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@650740c5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-42529-hadoop-hdfs-3_4_1-tests_jar-_-any-3382557061127057157/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:10:02,542 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a99c341{HTTP/1.1, (http/1.1)}{localhost:42529}
2024-11-10T02:10:02,542 INFO  [Time-limited test {}] server.Server(415): Started @160018ms
2024-11-10T02:10:02,553 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:10:02,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:02,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:02,803 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:02,808 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:02,809 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:02,809 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:02,810 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:10:02,810 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@623a52f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:02,811 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34c1099f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:02,949 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1935e32{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-41733-hadoop-hdfs-3_4_1-tests_jar-_-any-11464827672254977134/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:02,949 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@31e2f2e1{HTTP/1.1, (http/1.1)}{localhost:41733}
2024-11-10T02:10:02,950 INFO  [Time-limited test {}] server.Server(415): Started @160426ms
2024-11-10T02:10:02,952 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:02,995 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:03,000 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:03,004 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:03,004 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:03,005 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:03,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25cefabc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:03,006 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1118a265{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:03,125 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3db1f6e1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-41049-hadoop-hdfs-3_4_1-tests_jar-_-any-14705960574651300069/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:03,125 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1be3b2f7{HTTP/1.1, (http/1.1)}{localhost:41049}
2024-11-10T02:10:03,126 INFO  [Time-limited test {}] server.Server(415): Started @160601ms
2024-11-10T02:10:03,127 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:03,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:03,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:03,957 WARN  [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data2/current/BP-1556843191-172.17.0.3-1731204601934/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:03,957 WARN  [Thread-1216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data1/current/BP-1556843191-172.17.0.3-1731204601934/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:03,977 WARN  [Thread-1180 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:03,980 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x735feeb533c583ec with lease ID 0x29ad3aff62644dd8: Processing first storage report for DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959 from datanode DatanodeRegistration(127.0.0.1:46143, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=37779, infoSecurePort=0, ipcPort=40167, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934)
2024-11-10T02:10:03,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x735feeb533c583ec with lease ID 0x29ad3aff62644dd8: from storage DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959 node DatanodeRegistration(127.0.0.1:46143, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=37779, infoSecurePort=0, ipcPort=40167, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:03,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x735feeb533c583ec with lease ID 0x29ad3aff62644dd8: Processing first storage report for DS-3468d28d-d01a-4e2f-9b20-e36eb11071d8 from datanode DatanodeRegistration(127.0.0.1:46143, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=37779, infoSecurePort=0, ipcPort=40167, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934)
2024-11-10T02:10:03,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x735feeb533c583ec with lease ID 0x29ad3aff62644dd8: from storage DS-3468d28d-d01a-4e2f-9b20-e36eb11071d8 node DatanodeRegistration(127.0.0.1:46143, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=37779, infoSecurePort=0, ipcPort=40167, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:04,095 WARN  [Thread-1227 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data3/current/BP-1556843191-172.17.0.3-1731204601934/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:04,095 WARN  [Thread-1228 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data4/current/BP-1556843191-172.17.0.3-1731204601934/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:04,116 WARN  [Thread-1203 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:04,120 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f7b32a56c341683 with lease ID 0x29ad3aff62644dd9: Processing first storage report for DS-7d219750-c280-49c8-b26f-018f186fb526 from datanode DatanodeRegistration(127.0.0.1:37439, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=39899, infoSecurePort=0, ipcPort=33583, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934)
2024-11-10T02:10:04,120 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f7b32a56c341683 with lease ID 0x29ad3aff62644dd9: from storage DS-7d219750-c280-49c8-b26f-018f186fb526 node DatanodeRegistration(127.0.0.1:37439, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=39899, infoSecurePort=0, ipcPort=33583, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:04,120 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7f7b32a56c341683 with lease ID 0x29ad3aff62644dd9: Processing first storage report for DS-6905aeea-bad3-4e2a-b391-a6c18bac889c from datanode DatanodeRegistration(127.0.0.1:37439, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=39899, infoSecurePort=0, ipcPort=33583, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934)
2024-11-10T02:10:04,120 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7f7b32a56c341683 with lease ID 0x29ad3aff62644dd9: from storage DS-6905aeea-bad3-4e2a-b391-a6c18bac889c node DatanodeRegistration(127.0.0.1:37439, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=39899, infoSecurePort=0, ipcPort=33583, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:04,159 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118
2024-11-10T02:10:04,162 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/zookeeper_0, clientPort=59067, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:10:04,163 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59067
2024-11-10T02:10:04,164 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,165 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:10:04,179 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed with version=8
2024-11-10T02:10:04,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:10:04,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:10:04,181 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:10:04,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,182 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:10:04,182 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,182 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:10:04,182 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:10:04,182 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:10:04,183 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38109
2024-11-10T02:10:04,185 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38109 connecting to ZooKeeper ensemble=127.0.0.1:59067
2024-11-10T02:10:04,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:381090x0, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:10:04,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38109-0x1012281dcad0000 connected
2024-11-10T02:10:04,363 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,364 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,367 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:04,367 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed, hbase.cluster.distributed=false
2024-11-10T02:10:04,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:10:04,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38109
2024-11-10T02:10:04,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38109
2024-11-10T02:10:04,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38109
2024-11-10T02:10:04,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38109
2024-11-10T02:10:04,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38109
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:10:04,397 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:10:04,398 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33431
2024-11-10T02:10:04,401 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33431 connecting to ZooKeeper ensemble=127.0.0.1:59067
2024-11-10T02:10:04,401 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,404 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,419 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334310x0, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:10:04,419 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33431-0x1012281dcad0001 connected
2024-11-10T02:10:04,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:04,420 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:10:04,421 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:10:04,422 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:10:04,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:10:04,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-10T02:10:04,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33431
2024-11-10T02:10:04,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33431
2024-11-10T02:10:04,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-10T02:10:04,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-10T02:10:04,449 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:38109
2024-11-10T02:10:04,451 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,461 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:04,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:04,461 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,471 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:10:04,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,471 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,472 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:10:04,472 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,38109,1731204604181 from backup master directory
2024-11-10T02:10:04,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,530 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:04,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:04,531 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:10:04,531 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,536 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/hbase.id] with ID: 8edf7df8-4baa-43c7-b1f3-3e94c2211fd5
2024-11-10T02:10:04,536 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/.tmp/hbase.id
2024-11-10T02:10:04,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:10:04,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:10:04,544 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/.tmp/hbase.id]:[hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/hbase.id]
2024-11-10T02:10:04,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:04,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:10:04,560 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-10T02:10:04,572 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:10:04,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:10:04,581 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:10:04,582 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:10:04,582 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:04,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:10:04,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:10:04,591 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store
2024-11-10T02:10:04,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:10:04,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:10:04,607 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:04,607 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:10:04,607 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:04,607 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:04,608 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:10:04,608 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:04,608 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:04,608 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204604607Disabling compacts and flushes for region at 1731204604607Disabling writes for close at 1731204604608 (+1 ms)Writing region close event to WAL at 1731204604608Closed at 1731204604608
2024-11-10T02:10:04,613 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/.initializing
2024-11-10T02:10:04,613 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,616 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C38109%2C1731204604181, suffix=, logDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181, archiveDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/oldWALs, maxLogs=10
2024-11-10T02:10:04,616 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C38109%2C1731204604181.1731204604616
2024-11-10T02:10:04,626 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616
2024-11-10T02:10:04,635 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39899:39899),(127.0.0.1/127.0.0.1:37779:37779)]
2024-11-10T02:10:04,636 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:04,636 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:04,636 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,636 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,638 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:04,640 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:10:04,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,640 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:04,641 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:04,642 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:10:04,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,643 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:04,643 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:04,645 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:10:04,645 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,646 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:04,646 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:04,648 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:10:04,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,649 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:04,649 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,650 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,651 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,652 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,652 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,653 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:10:04,654 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:04,657 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:04,658 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872926, jitterRate=0.10998299717903137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:10:04,659 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204604636Initializing all the Stores at 1731204604637 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204604638 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204604638Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204604638Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204604638Cleaning up temporary data from old regions at 1731204604652 (+14 ms)Region opened successfully at 1731204604659 (+7 ms)
2024-11-10T02:10:04,659 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:10:04,664 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5245b6a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:10:04,665 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:10:04,665 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:10:04,665 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:10:04,665 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:10:04,666 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:10:04,667 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:10:04,667 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:10:04,672 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:10:04,673 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:10:04,682 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:10:04,682 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:10:04,683 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:10:04,692 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:10:04,693 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:10:04,694 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:10:04,703 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:10:04,704 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:10:04,713 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:10:04,715 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:10:04,724 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:10:04,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:04,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:04,734 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:04,734 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,735 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,38109,1731204604181, sessionid=0x1012281dcad0000, setting cluster-up flag (Was=false)
2024-11-10T02:10:04,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:04,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,755 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,787 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:10:04,788 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,808 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:04,840 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:10:04,841 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:04,842 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:10:04,844 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:04,844 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:10:04,844 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:10:04,845 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,38109,1731204604181
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:10:04,846 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:04,847 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204634847
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:10:04,848 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:04,849 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:10:04,849 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:04,849 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:10:04,849 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:10:04,849 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:10:04,849 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:10:04,849 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:10:04,849 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204604849,5,FailOnTimeoutGroup]
2024-11-10T02:10:04,850 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204604850,5,FailOnTimeoutGroup]
2024-11-10T02:10:04,850 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:04,850 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:10:04,850 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:04,850 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:04,850 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,850 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:10:04,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:10:04,862 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:10:04,864 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:10:04,864 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed
2024-11-10T02:10:04,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:10:04,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:10:04,875 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:04,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:10:04,879 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:10:04,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:04,880 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:10:04,881 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:10:04,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,882 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:04,882 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:10:04,883 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:10:04,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,884 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:04,884 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:10:04,886 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:10:04,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:04,887 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:04,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:10:04,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740
2024-11-10T02:10:04,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740
2024-11-10T02:10:04,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:10:04,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:10:04,891 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:10:04,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:10:04,894 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:04,895 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821101, jitterRate=0.04408465325832367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:10:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204604875Initializing all the Stores at 1731204604876 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204604877 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204604877Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204604877Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204604877Cleaning up temporary data from old regions at 1731204604890 (+13 ms)Region opened successfully at 1731204604896 (+6 ms)
2024-11-10T02:10:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:10:04,896 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:10:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:10:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:10:04,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:10:04,897 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:10:04,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204604896Disabling compacts and flushes for region at 1731204604896Disabling writes for close at 1731204604896Writing region close event to WAL at 1731204604897 (+1 ms)Closed at 1731204604897
2024-11-10T02:10:04,898 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:04,899 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:10:04,899 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:10:04,901 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:10:04,902 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:10:04,934 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(746): ClusterId : 8edf7df8-4baa-43c7-b1f3-3e94c2211fd5
2024-11-10T02:10:04,934 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:10:04,946 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:10:04,946 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:10:04,956 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:10:04,957 DEBUG [RS:0;0c4d4d227d60:33431 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2190bf19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:10:04,974 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:33431
2024-11-10T02:10:04,974 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:10:04,974 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:10:04,974 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:10:04,975 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,38109,1731204604181 with port=33431, startcode=1731204604396
2024-11-10T02:10:04,975 DEBUG [RS:0;0c4d4d227d60:33431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:10:04,977 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:10:04,978 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:04,978 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38109 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:04,980 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed
2024-11-10T02:10:04,980 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41207
2024-11-10T02:10:04,980 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:10:04,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:10:04,987 DEBUG [RS:0;0c4d4d227d60:33431 {}] zookeeper.ZKUtil(111): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:04,988 WARN  [RS:0;0c4d4d227d60:33431 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:10:04,988 INFO  [RS:0;0c4d4d227d60:33431 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:04,988 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:04,988 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,33431,1731204604396]
2024-11-10T02:10:04,995 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:10:05,000 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:10:05,001 INFO  [RS:0;0c4d4d227d60:33431 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:10:05,001 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,001 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:10:05,002 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:10:05,002 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,002 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,002 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,002 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:10:05,003 DEBUG [RS:0;0c4d4d227d60:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,008 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33431,1731204604396-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:10:05,031 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:10:05,031 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33431,1731204604396-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,031 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,032 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.Replication(171): 0c4d4d227d60,33431,1731204604396 started
2024-11-10T02:10:05,052 WARN  [0c4d4d227d60:38109 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:10:05,052 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,052 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,33431,1731204604396, RpcServer on 0c4d4d227d60/172.17.0.3:33431, sessionid=0x1012281dcad0001
2024-11-10T02:10:05,053 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:10:05,053 DEBUG [RS:0;0c4d4d227d60:33431 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,053 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,33431,1731204604396'
2024-11-10T02:10:05,053 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:10:05,053 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,33431,1731204604396'
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:10:05,054 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:10:05,055 DEBUG [RS:0;0c4d4d227d60:33431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:10:05,055 INFO  [RS:0;0c4d4d227d60:33431 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:10:05,055 INFO  [RS:0;0c4d4d227d60:33431 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:10:05,157 INFO  [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C33431%2C1731204604396, suffix=, logDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396, archiveDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs, maxLogs=32
2024-11-10T02:10:05,157 INFO  [RS:0;0c4d4d227d60:33431 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:05,164 INFO  [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:05,168 DEBUG [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39899:39899),(127.0.0.1/127.0.0.1:37779:37779)]
2024-11-10T02:10:05,302 DEBUG [0c4d4d227d60:38109 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:10:05,303 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,304 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,33431,1731204604396, state=OPENING
2024-11-10T02:10:05,313 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:10:05,324 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:05,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:05,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:05,325 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:05,325 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:10:05,325 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,33431,1731204604396}]
2024-11-10T02:10:05,479 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:10:05,480 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47829, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:10:05,484 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:10:05,485 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:05,486 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C33431%2C1731204604396.meta, suffix=.meta, logDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396, archiveDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs, maxLogs=32
2024-11-10T02:10:05,487 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta
2024-11-10T02:10:05,495 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta
2024-11-10T02:10:05,497 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39899:39899),(127.0.0.1/127.0.0.1:37779:37779)]
2024-11-10T02:10:05,497 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:10:05,498 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:10:05,498 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:10:05,499 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:10:05,500 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:10:05,500 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:05,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:05,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:10:05,502 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:10:05,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:05,502 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:05,502 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:10:05,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:10:05,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:05,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:05,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:10:05,505 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:10:05,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:05,505 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:05,506 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:10:05,507 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740
2024-11-10T02:10:05,508 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740
2024-11-10T02:10:05,509 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:10:05,509 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:10:05,510 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:10:05,512 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:10:05,513 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=865674, jitterRate=0.10076190531253815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:10:05,513 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:10:05,514 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204605498Writing region info on filesystem at 1731204605498Initializing all the Stores at 1731204605499 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204605499Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204605499Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204605499Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204605499Cleaning up temporary data from old regions at 1731204605509 (+10 ms)Running coprocessor post-open hooks at 1731204605513 (+4 ms)Region opened successfully at 1731204605514 (+1 ms)
2024-11-10T02:10:05,515 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204605478
2024-11-10T02:10:05,517 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:10:05,517 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:10:05,518 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,519 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,33431,1731204604396, state=OPEN
2024-11-10T02:10:05,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:10:05,587 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:10:05,587 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:05,587 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:05,589 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:10:05,590 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,33431,1731204604396 in 262 msec
2024-11-10T02:10:05,592 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:10:05,592 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 691 msec
2024-11-10T02:10:05,593 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:05,593 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:10:05,594 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:10:05,594 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,33431,1731204604396, seqNum=-1]
2024-11-10T02:10:05,595 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:10:05,596 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53711, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:10:05,602 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 758 msec
2024-11-10T02:10:05,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204605603, completionTime=-1
2024-11-10T02:10:05,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:10:05,603 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204665605
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204725605
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:38109, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,606 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,607 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:10:05,609 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.078sec
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:10:05,610 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:10:05,613 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:10:05,613 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:10:05,613 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,38109,1731204604181-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:05,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a7e8687, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:05,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,38109,-1 for getting cluster id
2024-11-10T02:10:05,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:10:05,638 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8edf7df8-4baa-43c7-b1f3-3e94c2211fd5'
2024-11-10T02:10:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:10:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8edf7df8-4baa-43c7-b1f3-3e94c2211fd5"

2024-11-10T02:10:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169e4cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,38109,-1]
2024-11-10T02:10:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:10:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:05,642 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:10:05,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b81225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:05,643 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:10:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,33431,1731204604396, seqNum=-1]
2024-11-10T02:10:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:10:05,647 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:10:05,650 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:05,651 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:05,655 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:10:05,655 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-10T02:10:05,655 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-10T02:10:05,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-10T02:10:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2cbba9b4
2024-11-10T02:10:05,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-10T02:10:05,660 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-10T02:10:05,660 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-10T02:10:05,660 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-10T02:10:05,661 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:10:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-10T02:10:05,665 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-10T02:10:05,665 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:05,665 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-10T02:10:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:10:05,667 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-10T02:10:05,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741835_1011 (size=395)
2024-11-10T02:10:05,681 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eb05020078394171f853cc0a0f9a6d47, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed
2024-11-10T02:10:05,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741835_1011 (size=395)
2024-11-10T02:10:05,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37439 is added to blk_1073741836_1012 (size=78)
2024-11-10T02:10:05,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46143 is added to blk_1073741836_1012 (size=78)
2024-11-10T02:10:05,697 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:05,697 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing eb05020078394171f853cc0a0f9a6d47, disabling compactions & flushes
2024-11-10T02:10:05,697 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:05,698 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:05,698 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47. after waiting 0 ms
2024-11-10T02:10:05,698 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:05,698 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:05,698 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for eb05020078394171f853cc0a0f9a6d47:
Waiting for close lock at 1731204605697Disabling compacts and flushes for region at 1731204605697Disabling writes for close at 1731204605698 (+1 ms)Writing region close event to WAL at 1731204605698Closed at 1731204605698
2024-11-10T02:10:05,700 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-10T02:10:05,700 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731204605700"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204605700"}]},"ts":"1731204605700"}
2024-11-10T02:10:05,703 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-10T02:10:05,704 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-10T02:10:05,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204605704"}]},"ts":"1731204605704"}
2024-11-10T02:10:05,707 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-10T02:10:05,708 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=eb05020078394171f853cc0a0f9a6d47, ASSIGN}]
2024-11-10T02:10:05,710 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=eb05020078394171f853cc0a0f9a6d47, ASSIGN
2024-11-10T02:10:05,711 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=eb05020078394171f853cc0a0f9a6d47, ASSIGN; state=OFFLINE, location=0c4d4d227d60,33431,1731204604396; forceNewPlan=false, retain=false
2024-11-10T02:10:05,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:05,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:05,866 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb05020078394171f853cc0a0f9a6d47, regionState=OPENING, regionLocation=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:05,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=eb05020078394171f853cc0a0f9a6d47, ASSIGN because future has completed
2024-11-10T02:10:05,870 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb05020078394171f853cc0a0f9a6d47, server=0c4d4d227d60,33431,1731204604396}]
2024-11-10T02:10:06,028 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:06,028 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eb05020078394171f853cc0a0f9a6d47, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,031 INFO  [StoreOpener-eb05020078394171f853cc0a0f9a6d47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region eb05020078394171f853cc0a0f9a6d47 
2024-11-10T02:10:06,033 INFO  [StoreOpener-eb05020078394171f853cc0a0f9a6d47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb05020078394171f853cc0a0f9a6d47 columnFamilyName info
2024-11-10T02:10:06,033 DEBUG [StoreOpener-eb05020078394171f853cc0a0f9a6d47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:06,034 INFO  [StoreOpener-eb05020078394171f853cc0a0f9a6d47-1 {}] regionserver.HStore(327): Store=eb05020078394171f853cc0a0f9a6d47/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:06,034 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,035 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,035 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,035 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,035 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,037 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,041 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:06,042 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eb05020078394171f853cc0a0f9a6d47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800431, jitterRate=0.01780146360397339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:10:06,042 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:06,043 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eb05020078394171f853cc0a0f9a6d47:
Running coprocessor pre-open hook at 1731204606029Writing region info on filesystem at 1731204606029Initializing all the Stores at 1731204606031 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204606031Cleaning up temporary data from old regions at 1731204606035 (+4 ms)Running coprocessor post-open hooks at 1731204606042 (+7 ms)Region opened successfully at 1731204606043 (+1 ms)
2024-11-10T02:10:06,044 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47., pid=6, masterSystemTime=1731204606024
2024-11-10T02:10:06,047 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:06,047 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:06,048 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eb05020078394171f853cc0a0f9a6d47, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:06,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb05020078394171f853cc0a0f9a6d47, server=0c4d4d227d60,33431,1731204604396 because future has completed
2024-11-10T02:10:06,055 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-10T02:10:06,055 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eb05020078394171f853cc0a0f9a6d47, server=0c4d4d227d60,33431,1731204604396 in 182 msec
2024-11-10T02:10:06,058 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-10T02:10:06,058 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=eb05020078394171f853cc0a0f9a6d47, ASSIGN in 347 msec
2024-11-10T02:10:06,060 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-10T02:10:06,060 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204606060"}]},"ts":"1731204606060"}
2024-11-10T02:10:06,063 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-10T02:10:06,064 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-10T02:10:06,067 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 403 msec
2024-11-10T02:10:06,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:06,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:07,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:07,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:08,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:08,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:09,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:10:09,422 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-10T02:10:09,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-10T02:10:09,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-10T02:10:09,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:10:09,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-10T02:10:09,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:09,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:10,508 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,508 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,508 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,509 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,509 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,509 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,513 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,513 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,513 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,515 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:10,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:10,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:11,019 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:10:11,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,037 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,042 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,042 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,042 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:11,053 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-10T02:10:11,053 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-10T02:10:11,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:11,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:12,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:12,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:13,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:13,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:14,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:14,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:15,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:10:15,711 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-10T02:10:15,711 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-10T02:10:15,714 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-10T02:10:15,714 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47., hostname=0c4d4d227d60,33431,1731204604396, seqNum=2]
2024-11-10T02:10:15,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:15,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:16,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:16,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:17,720 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:17,721 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,721 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,721 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK], DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]) is bad.
2024-11-10T02:10:17,721 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK], DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]) is bad.
2024-11-10T02:10:17,722 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,722 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK], DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37439,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]) is bad.
2024-11-10T02:10:17,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_188591216_22 at /127.0.0.1:55208 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37439:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55208 dst: /127.0.0.1:37439
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:55242 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37439:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55242 dst: /127.0.0.1:37439
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:55230 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37439:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55230 dst: /127.0.0.1:37439
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_188591216_22 at /127.0.0.1:35054 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35054 dst: /127.0.0.1:46143
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,723 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:35114 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35114 dst: /127.0.0.1:46143
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,723 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:35098 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35098 dst: /127.0.0.1:46143
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:17,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:17,781 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3db1f6e1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:17,782 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1be3b2f7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:17,782 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:17,782 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1118a265{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:17,782 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25cefabc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:17,784 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:17,785 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid 48ea7c40-b0f7-48ae-b452-7d5050a9edc0) service to localhost/127.0.0.1:41207
2024-11-10T02:10:17,785 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:17,785 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:17,785 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data3/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:17,786 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data4/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:17,786 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:17,799 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:17,803 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:17,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:17,804 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:17,804 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:17,808 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6614137b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:17,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ff7fcbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:17,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20432799{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-42015-hadoop-hdfs-3_4_1-tests_jar-_-any-3563779860394838246/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:17,914 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22d0d1b0{HTTP/1.1, (http/1.1)}{localhost:42015}
2024-11-10T02:10:17,914 INFO  [Time-limited test {}] server.Server(415): Started @175390ms
2024-11-10T02:10:17,916 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:17,944 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,944 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,944 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:17,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:51284 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51284 dst: /127.0.0.1:46143
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_188591216_22 at /127.0.0.1:51266 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51266 dst: /127.0.0.1:46143
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,946 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:51270 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51270 dst: /127.0.0.1:46143
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:17,950 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1935e32{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:17,950 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@31e2f2e1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:17,951 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:17,951 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34c1099f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:17,951 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@623a52f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:17,956 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:17,956 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8) service to localhost/127.0.0.1:41207
2024-11-10T02:10:17,957 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data1/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:17,957 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data2/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:17,957 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:17,957 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:17,957 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:17,971 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:17,975 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:17,980 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:17,980 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:17,980 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:17,981 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e8914c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:17,981 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a441552{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:18,080 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1decdda3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-41209-hadoop-hdfs-3_4_1-tests_jar-_-any-4238795197547414378/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:18,080 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c02d1f3{HTTP/1.1, (http/1.1)}{localhost:41209}
2024-11-10T02:10:18,080 INFO  [Time-limited test {}] server.Server(415): Started @175556ms
2024-11-10T02:10:18,082 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:18,524 WARN  [Thread-1351 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:18,526 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a596659dc6126a4 with lease ID 0x29ad3aff62644dda: from storage DS-7d219750-c280-49c8-b26f-018f186fb526 node DatanodeRegistration(127.0.0.1:33563, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=38601, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:18,527 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a596659dc6126a4 with lease ID 0x29ad3aff62644dda: from storage DS-6905aeea-bad3-4e2a-b391-a6c18bac889c node DatanodeRegistration(127.0.0.1:33563, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=38601, infoSecurePort=0, ipcPort=38637, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:18,637 WARN  [Thread-1371 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:18,639 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2e434121b517466 with lease ID 0x29ad3aff62644ddb: from storage DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959 node DatanodeRegistration(127.0.0.1:36235, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=34593, infoSecurePort=0, ipcPort=41613, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:18,640 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2e434121b517466 with lease ID 0x29ad3aff62644ddb: from storage DS-3468d28d-d01a-4e2f-9b20-e36eb11071d8 node DatanodeRegistration(127.0.0.1:36235, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=34593, infoSecurePort=0, ipcPort=41613, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:18,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:18,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:19,114 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-10T02:10:19,116 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-10T02:10:19,117 ERROR [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:19,118 WARN  [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:19,118 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C33431%2C1731204604396:(num 1731204605157) roll requested
2024-11-10T02:10:19,118 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:19,123 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 newFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:19,124 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:19,124 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:19,124 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:19,124 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:19,124 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:19,124 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:19,125 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:19,125 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:19,125 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:19,125 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38601:38601),(127.0.0.1/127.0.0.1:34593:34593)]
2024-11-10T02:10:19,125 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 is not closed yet, will try archiving it next time
2024-11-10T02:10:19,125 WARN  [IPC Server handler 1 on default port 41207 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-10T02:10:19,126 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 after 1ms
2024-11-10T02:10:19,735 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:19,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:20,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:20,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:21,128 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-10T02:10:21,527 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-10T02:10:21,736 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:21,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:22,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:22,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:23,126 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 after 4001ms
2024-11-10T02:10:23,131 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:36235,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:23,131 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33563,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK], DatanodeInfoWithStorage[127.0.0.1:36235,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36235,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]) is bad.
2024-11-10T02:10:23,131 WARN  [PacketResponder: BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36235] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:23,131 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:39168 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33563:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39168 dst: /127.0.0.1:33563
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:23,132 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:49116 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36235:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49116 dst: /127.0.0.1:36235
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:23,178 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1decdda3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:23,178 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c02d1f3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:23,178 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:23,179 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a441552{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:23,179 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e8914c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:23,180 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:23,180 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:23,180 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:23,180 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8) service to localhost/127.0.0.1:41207
2024-11-10T02:10:23,180 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data1/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:23,181 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data2/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:23,181 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:23,190 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:23,193 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:23,194 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:23,195 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:23,195 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:10:23,195 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c0d41d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:23,196 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76a8c1ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:23,291 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d867aab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-43101-hadoop-hdfs-3_4_1-tests_jar-_-any-1059260859825939758/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:23,291 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a9d36bb{HTTP/1.1, (http/1.1)}{localhost:43101}
2024-11-10T02:10:23,291 INFO  [Time-limited test {}] server.Server(415): Started @180767ms
2024-11-10T02:10:23,292 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:23,310 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:23,310 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1386152324_22 at /127.0.0.1:51718 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33563:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51718 dst: /127.0.0.1:33563
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:23,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20432799{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:23,315 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22d0d1b0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:23,315 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:23,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ff7fcbc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:23,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6614137b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:23,316 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:23,316 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:23,317 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:23,317 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid 48ea7c40-b0f7-48ae-b452-7d5050a9edc0) service to localhost/127.0.0.1:41207
2024-11-10T02:10:23,317 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data3/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:23,318 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data4/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:23,318 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:23,331 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:23,334 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:23,336 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:23,336 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:23,336 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:23,338 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cbccde7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:23,339 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27b0e5fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:23,434 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@574f69be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/java.io.tmpdir/jetty-localhost-34789-hadoop-hdfs-3_4_1-tests_jar-_-any-12774832253842406964/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:23,434 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2da9d855{HTTP/1.1, (http/1.1)}{localhost:34789}
2024-11-10T02:10:23,434 INFO  [Time-limited test {}] server.Server(415): Started @180910ms
2024-11-10T02:10:23,435 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:23,737 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:23,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:23,838 WARN  [Thread-1425 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:23,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc05a6255fbe9688a with lease ID 0x29ad3aff62644ddc: from storage DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=44743, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:23,841 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc05a6255fbe9688a with lease ID 0x29ad3aff62644ddc: from storage DS-3468d28d-d01a-4e2f-9b20-e36eb11071d8 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8, infoPort=44743, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:24,012 WARN  [Thread-1445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:24,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8a598d72534350 with lease ID 0x29ad3aff62644ddd: from storage DS-7d219750-c280-49c8-b26f-018f186fb526 node DatanodeRegistration(127.0.0.1:34645, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=35403, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:10:24,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe8a598d72534350 with lease ID 0x29ad3aff62644ddd: from storage DS-6905aeea-bad3-4e2a-b391-a6c18bac889c node DatanodeRegistration(127.0.0.1:34645, datanodeUuid=48ea7c40-b0f7-48ae-b452-7d5050a9edc0, infoPort=35403, infoSecurePort=0, ipcPort=44121, storageInfo=lv=-57;cid=testClusterID;nsid=830871995;c=1731204601934), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:24,452 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-10T02:10:24,454 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-10T02:10:24,455 ERROR [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33563,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:24,455 WARN  [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33563,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:24,455 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C33431%2C1731204604396:(num 1731204619118) roll requested
2024-11-10T02:10:24,455 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:24,460 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 newFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:24,461 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:24,461 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:24,461 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:24,461 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:24,461 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:24,461 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:24,461 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33563,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:24,462 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33563,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:24,462 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:24,462 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35403:35403),(127.0.0.1/127.0.0.1:44743:44743)]
2024-11-10T02:10:24,462 WARN  [IPC Server handler 3 on default port 41207 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-10T02:10:24,462 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 is not closed yet, will try archiving it next time
2024-11-10T02:10:24,463 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 after 1ms
2024-11-10T02:10:24,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:24,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:25,738 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:25,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:26,464 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:26,469 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455 newFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:26,469 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:26,469 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:26,469 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:26,469 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:26,469 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:26,470 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:26,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44743:44743),(127.0.0.1/127.0.0.1:35403:35403)]
2024-11-10T02:10:26,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 is not closed yet, will try archiving it next time
2024-11-10T02:10:26,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455 is not closed yet, will try archiving it next time
2024-11-10T02:10:26,471 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:26,471 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:26,471 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157 after 0ms
2024-11-10T02:10:26,471 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:26,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741838_1019 (size=1264)
2024-11-10T02:10:26,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741838_1019 (size=1264)
2024-11-10T02:10:26,472 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 is not closed yet, will try archiving it next time
2024-11-10T02:10:26,479 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731204606043/Put/vlen=218/seqid=0]
2024-11-10T02:10:26,480 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731204615718/Put/vlen=1045/seqid=0]
2024-11-10T02:10:26,480 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204605157
2024-11-10T02:10:26,480 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:26,480 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:26,480 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 after 0ms
2024-11-10T02:10:26,480 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:26,484 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731204619117/Put/vlen=1045/seqid=0]
2024-11-10T02:10:26,484 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731204621129/Put/vlen=1045/seqid=0]
2024-11-10T02:10:26,484 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118
2024-11-10T02:10:26,484 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:26,484 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:26,484 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455 after 0ms
2024-11-10T02:10:26,484 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204624455
2024-11-10T02:10:26,487 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731204624455/Put/vlen=1045/seqid=0]
2024-11-10T02:10:26,487 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:26,487 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:26,488 WARN  [IPC Server handler 0 on default port 41207 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-10T02:10:26,488 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 after 1ms
2024-11-10T02:10:26,739 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:26,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:26,841 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-10T02:10:27,017 WARN  [ResponseProcessor for block BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:27,017 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_188591216_22 at /127.0.0.1:55556 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35109:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55556 dst: /127.0.0.1:35109
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35109 remote=/127.0.0.1:55556]. Total timeout mills is 60000, 59451 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:27,018 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_188591216_22 at /127.0.0.1:47074 [Receiving block BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34645:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47074 dst: /127.0.0.1:34645
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:10:27,018 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 block BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35109,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK], DatanodeInfoWithStorage[127.0.0.1:34645,DS-7d219750-c280-49c8-b26f-018f186fb526,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35109,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]) is bad.
2024-11-10T02:10:27,019 WARN  [DataStreamer for file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 block BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:27,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741839_1022 (size=85)
2024-11-10T02:10:27,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:27,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:28,464 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204619118 after 4002ms
2024-11-10T02:10:28,740 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:28,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:29,741 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:29,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:30,489 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 after 4002ms
2024-11-10T02:10:30,489 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:30,494 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:30,494 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing eb05020078394171f853cc0a0f9a6d47 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-10T02:10:30,494 ERROR [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,495 WARN  [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,495 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C33431%2C1731204604396:(num 1731204626463) roll requested
2024-11-10T02:10:30,496 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.1731204630495
2024-11-10T02:10:30,505 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 newFile=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204630495
2024-11-10T02:10:30,505 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,505 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,506 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,506 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,506 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,506 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204630495
2024-11-10T02:10:30,506 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,506 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1556843191-172.17.0.3-1731204601934:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,507 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:30,507 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35403:35403),(127.0.0.1/127.0.0.1:44743:44743)]
2024-11-10T02:10:30,507 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 is not closed yet, will try archiving it next time
2024-11-10T02:10:30,508 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 after 1ms
2024-11-10T02:10:30,508 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.1731204626463 to hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs/0c4d4d227d60%2C33431%2C1731204604396.1731204626463
2024-11-10T02:10:30,532 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/.tmp/info/372792cfd663430d893ca11d79c905b1 is 1080, key is row1002/info:/1731204615718/Put/seqid=0
2024-11-10T02:10:30,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741841_1024 (size=9270)
2024-11-10T02:10:30,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741841_1024 (size=9270)
2024-11-10T02:10:30,543 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/.tmp/info/372792cfd663430d893ca11d79c905b1
2024-11-10T02:10:30,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/.tmp/info/372792cfd663430d893ca11d79c905b1 as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/info/372792cfd663430d893ca11d79c905b1
2024-11-10T02:10:30,566 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/info/372792cfd663430d893ca11d79c905b1, entries=4, sequenceid=8, filesize=9.1 K
2024-11-10T02:10:30,568 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for eb05020078394171f853cc0a0f9a6d47 in 74ms, sequenceid=8, compaction requested=false
2024-11-10T02:10:30,568 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eb05020078394171f853cc0a0f9a6d47:

2024-11-10T02:10:30,569 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-10T02:10:30,569 ERROR [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,569 WARN  [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed-prefix:0c4d4d227d60,33431,1731204604396.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,569 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C33431%2C1731204604396.meta:.meta(num 1731204605487) roll requested
2024-11-10T02:10:30,570 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33431%2C1731204604396.meta.1731204630570.meta
2024-11-10T02:10:30,586 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,587 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,587 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,587 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,587 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:30,587 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204630570.meta
2024-11-10T02:10:30,588 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,588 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:30,588 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta
2024-11-10T02:10:30,589 WARN  [IPC Server handler 4 on default port 41207 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1015
2024-11-10T02:10:30,589 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta after 1ms
2024-11-10T02:10:30,596 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44743:44743),(127.0.0.1/127.0.0.1:35403:35403)]
2024-11-10T02:10:30,596 DEBUG [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta is not closed yet, will try archiving it next time
2024-11-10T02:10:30,623 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/info/86c38e6a01c64d359acf7d9af485168b is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47./info:regioninfo/1731204606048/Put/seqid=0
2024-11-10T02:10:30,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741843_1027 (size=7125)
2024-11-10T02:10:30,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741843_1027 (size=7125)
2024-11-10T02:10:30,642 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/info/86c38e6a01c64d359acf7d9af485168b
2024-11-10T02:10:30,674 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/ns/7864e616abe44ec5bb83eb97abd2b648 is 43, key is default/ns:d/1731204605597/Put/seqid=0
2024-11-10T02:10:30,687 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741844_1028 (size=5153)
2024-11-10T02:10:30,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741844_1028 (size=5153)
2024-11-10T02:10:30,688 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/ns/7864e616abe44ec5bb83eb97abd2b648
2024-11-10T02:10:30,719 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/table/a271e624c1844273bb92c2f0405e483c is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731204606060/Put/seqid=0
2024-11-10T02:10:30,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741845_1029 (size=5438)
2024-11-10T02:10:30,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741845_1029 (size=5438)
2024-11-10T02:10:30,726 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/table/a271e624c1844273bb92c2f0405e483c
2024-11-10T02:10:30,734 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/info/86c38e6a01c64d359acf7d9af485168b as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/info/86c38e6a01c64d359acf7d9af485168b
2024-11-10T02:10:30,742 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/info/86c38e6a01c64d359acf7d9af485168b, entries=10, sequenceid=11, filesize=7.0 K
2024-11-10T02:10:30,742 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:30,744 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/ns/7864e616abe44ec5bb83eb97abd2b648 as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/ns/7864e616abe44ec5bb83eb97abd2b648
2024-11-10T02:10:30,752 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/ns/7864e616abe44ec5bb83eb97abd2b648, entries=2, sequenceid=11, filesize=5.0 K
2024-11-10T02:10:30,753 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/.tmp/table/a271e624c1844273bb92c2f0405e483c as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/table/a271e624c1844273bb92c2f0405e483c
2024-11-10T02:10:30,759 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/table/a271e624c1844273bb92c2f0405e483c, entries=2, sequenceid=11, filesize=5.3 K
2024-11-10T02:10:30,760 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 192ms, sequenceid=11, compaction requested=false
2024-11-10T02:10:30,760 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-10T02:10:30,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:30,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:10:30,766 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:10:30,766 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:10:30,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:30,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:30,766 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:10:30,766 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:10:30,766 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=525396736, stopped=false
2024-11-10T02:10:30,767 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,38109,1731204604181
2024-11-10T02:10:30,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:30,881 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:30,881 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:30,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:30,881 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:10:30,881 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:10:30,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:10:30,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:30,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:30,881 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,33431,1731204604396' *****
2024-11-10T02:10:30,881 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:10:30,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:30,881 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:10:30,882 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(3091): Received CLOSE for eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:33431.
2024-11-10T02:10:30,882 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb05020078394171f853cc0a0f9a6d47, disabling compactions & flushes
2024-11-10T02:10:30,882 DEBUG [RS:0;0c4d4d227d60:33431 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:10:30,882 DEBUG [RS:0;0c4d4d227d60:33431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:30,882 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:10:30,882 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:30,883 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-10T02:10:30,883 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1325): Online Regions={eb05020078394171f853cc0a0f9a6d47=TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47., 1588230740=hbase:meta,,1.1588230740}
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47. after waiting 0 ms
2024-11-10T02:10:30,883 DEBUG [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eb05020078394171f853cc0a0f9a6d47
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:10:30,883 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:10:30,883 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:10:30,887 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-10T02:10:30,887 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/data/default/TestLogRolling-testLogRollOnPipelineRestart/eb05020078394171f853cc0a0f9a6d47/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-10T02:10:30,887 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:10:30,887 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:30,887 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:10:30,887 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204630883Running coprocessor pre-close hooks at 1731204630883Disabling compacts and flushes for region at 1731204630883Disabling writes for close at 1731204630883Writing region close event to WAL at 1731204630884 (+1 ms)Running coprocessor post-close hooks at 1731204630887 (+3 ms)Closed at 1731204630887
2024-11-10T02:10:30,887 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb05020078394171f853cc0a0f9a6d47:
Waiting for close lock at 1731204630882Running coprocessor pre-close hooks at 1731204630882Disabling compacts and flushes for region at 1731204630882Disabling writes for close at 1731204630883 (+1 ms)Writing region close event to WAL at 1731204630883Running coprocessor post-close hooks at 1731204630887 (+4 ms)Closed at 1731204630887
2024-11-10T02:10:30,888 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:10:30,888 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731204605660.eb05020078394171f853cc0a0f9a6d47.
2024-11-10T02:10:31,008 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:10:31,008 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:10:31,010 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:10:31,083 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,33431,1731204604396; all regions closed.
2024-11-10T02:10:31,083 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:31,084 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:31,084 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:31,084 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:31,084 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:31,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741842_1025 (size=825)
2024-11-10T02:10:31,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741842_1025 (size=825)
2024-11-10T02:10:31,743 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:31,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:32,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:32,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:33,016 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-10T02:10:33,744 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:33,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:34,158 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:10:34,590 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta after 4002ms
2024-11-10T02:10:34,591 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/WALs/0c4d4d227d60,33431,1731204604396/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta to hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs/0c4d4d227d60%2C33431%2C1731204604396.meta.1731204605487.meta
2024-11-10T02:10:34,594 DEBUG [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs
2024-11-10T02:10:34,594 INFO  [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C33431%2C1731204604396.meta:.meta(num 1731204630570)
2024-11-10T02:10:34,595 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,595 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,595 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,595 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,595 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741840_1023 (size=1162)
2024-11-10T02:10:34,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741840_1023 (size=1162)
2024-11-10T02:10:34,604 DEBUG [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C33431%2C1731204604396:(num 1731204630495)
2024-11-10T02:10:34,604 DEBUG [RS:0;0c4d4d227d60:33431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:10:34,604 INFO  [RS:0;0c4d4d227d60:33431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33431
2024-11-10T02:10:34,606 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:10:34,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:10:34,659 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,33431,1731204604396
2024-11-10T02:10:34,659 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:10:34,660 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,33431,1731204604396]
2024-11-10T02:10:34,680 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,33431,1731204604396 already deleted, retry=false
2024-11-10T02:10:34,680 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,33431,1731204604396 expired; onlineServers=0
2024-11-10T02:10:34,680 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,38109,1731204604181' *****
2024-11-10T02:10:34,680 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:10:34,680 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:10:34,681 DEBUG [M:0;0c4d4d227d60:38109 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:10:34,681 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:10:34,681 DEBUG [M:0;0c4d4d227d60:38109 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:10:34,681 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204604849 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204604849,5,FailOnTimeoutGroup]
2024-11-10T02:10:34,681 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204604850 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204604850,5,FailOnTimeoutGroup]
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:10:34,681 DEBUG [M:0;0c4d4d227d60:38109 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:10:34,681 INFO  [M:0;0c4d4d227d60:38109 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:10:34,682 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:10:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:10:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:34,691 DEBUG [M:0;0c4d4d227d60:38109 {}] zookeeper.ZKUtil(347): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:10:34,691 WARN  [M:0;0c4d4d227d60:38109 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:10:34,692 INFO  [M:0;0c4d4d227d60:38109 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/.lastflushedseqids
2024-11-10T02:10:34,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741846_1030 (size=111)
2024-11-10T02:10:34,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741846_1030 (size=111)
2024-11-10T02:10:34,703 INFO  [M:0;0c4d4d227d60:38109 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:10:34,703 INFO  [M:0;0c4d4d227d60:38109 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:10:34,703 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:10:34,703 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:34,703 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:34,703 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:10:34,703 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:34,703 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-10T02:10:34,704 ERROR [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData-prefix:0c4d4d227d60,38109,1731204604181 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:34,704 WARN  [FSHLog-0-hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData-prefix:0c4d4d227d60,38109,1731204604181 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:34,704 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0c4d4d227d60%2C38109%2C1731204604181:(num 1731204604616) roll requested
2024-11-10T02:10:34,705 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C38109%2C1731204604181.1731204634704
2024-11-10T02:10:34,712 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,713 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,713 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,713 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,713 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,713 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204634704
2024-11-10T02:10:34,713 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:34,714 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46143,DS-4d2813c8-0254-4e20-ab43-08dfc2f9d959,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-10T02:10:34,714 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616
2024-11-10T02:10:34,714 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44743:44743),(127.0.0.1/127.0.0.1:35403:35403)]
2024-11-10T02:10:34,714 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 is not closed yet, will try archiving it next time
2024-11-10T02:10:34,714 WARN  [IPC Server handler 1 on default port 41207 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-11-10T02:10:34,715 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 after 0ms
2024-11-10T02:10:34,737 DEBUG [M:0;0c4d4d227d60:38109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66ad9f1f544b40178ef82907da40ff55 is 82, key is hbase:meta,,1/info:regioninfo/1731204605518/Put/seqid=0
2024-11-10T02:10:34,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741848_1033 (size=5672)
2024-11-10T02:10:34,744 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741848_1033 (size=5672)
2024-11-10T02:10:34,745 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66ad9f1f544b40178ef82907da40ff55
2024-11-10T02:10:34,745 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:34,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:34,770 INFO  [RS:0;0c4d4d227d60:33431 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:10:34,770 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:34,770 INFO  [RS:0;0c4d4d227d60:33431 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,33431,1731204604396; zookeeper connection closed.
2024-11-10T02:10:34,770 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x1012281dcad0001, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:34,771 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14a9d4f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14a9d4f2
2024-11-10T02:10:34,771 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:10:34,774 DEBUG [M:0;0c4d4d227d60:38109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c4c0ef8538d427fa259123e2ebf00ca is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731204606066/Put/seqid=0
2024-11-10T02:10:34,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741849_1034 (size=6118)
2024-11-10T02:10:34,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741849_1034 (size=6118)
2024-11-10T02:10:34,782 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c4c0ef8538d427fa259123e2ebf00ca
2024-11-10T02:10:34,814 DEBUG [M:0;0c4d4d227d60:38109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc0b8c3c46574c45b553dff42f2e690b is 69, key is 0c4d4d227d60,33431,1731204604396/rs:state/1731204604978/Put/seqid=0
2024-11-10T02:10:34,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741850_1035 (size=5156)
2024-11-10T02:10:34,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741850_1035 (size=5156)
2024-11-10T02:10:34,821 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc0b8c3c46574c45b553dff42f2e690b
2024-11-10T02:10:34,847 DEBUG [M:0;0c4d4d227d60:38109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/882326d923e94b0c8e3c3ffa41a54c4a is 52, key is load_balancer_on/state:d/1731204605653/Put/seqid=0
2024-11-10T02:10:34,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741851_1036 (size=5056)
2024-11-10T02:10:34,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741851_1036 (size=5056)
2024-11-10T02:10:34,854 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/882326d923e94b0c8e3c3ffa41a54c4a
2024-11-10T02:10:34,865 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/66ad9f1f544b40178ef82907da40ff55 as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/66ad9f1f544b40178ef82907da40ff55
2024-11-10T02:10:34,871 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/66ad9f1f544b40178ef82907da40ff55, entries=8, sequenceid=56, filesize=5.5 K
2024-11-10T02:10:34,873 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c4c0ef8538d427fa259123e2ebf00ca as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c4c0ef8538d427fa259123e2ebf00ca
2024-11-10T02:10:34,879 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c4c0ef8538d427fa259123e2ebf00ca, entries=6, sequenceid=56, filesize=6.0 K
2024-11-10T02:10:34,880 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cc0b8c3c46574c45b553dff42f2e690b as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cc0b8c3c46574c45b553dff42f2e690b
2024-11-10T02:10:34,887 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cc0b8c3c46574c45b553dff42f2e690b, entries=1, sequenceid=56, filesize=5.0 K
2024-11-10T02:10:34,888 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/882326d923e94b0c8e3c3ffa41a54c4a as hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/882326d923e94b0c8e3c3ffa41a54c4a
2024-11-10T02:10:34,894 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/882326d923e94b0c8e3c3ffa41a54c4a, entries=1, sequenceid=56, filesize=4.9 K
2024-11-10T02:10:34,895 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 192ms, sequenceid=56, compaction requested=false
2024-11-10T02:10:34,897 INFO  [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:34,897 DEBUG [M:0;0c4d4d227d60:38109 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204634703Disabling compacts and flushes for region at 1731204634703Disabling writes for close at 1731204634703Obtaining lock to block concurrent updates at 1731204634703Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204634703Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731204634704 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204634715 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204634715Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204634736 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204634736Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204634752 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204634773 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204634773Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204634789 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204634813 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204634813Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204634828 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204634846 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204634846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a75808: reopening flushed file at 1731204634864 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60980371: reopening flushed file at 1731204634872 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@445a8d4e: reopening flushed file at 1731204634879 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e9a9ef8: reopening flushed file at 1731204634887 (+8 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 192ms, sequenceid=56, compaction requested=false at 1731204634895 (+8 ms)Writing region close event to WAL at 1731204634897 (+2 ms)Closed at 1731204634897
2024-11-10T02:10:34,897 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,897 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,897 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,898 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,898 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:10:34,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34645 is added to blk_1073741847_1031 (size=757)
2024-11-10T02:10:34,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741847_1031 (size=757)
2024-11-10T02:10:35,746 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:35,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:35,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,920 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,920 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,921 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:35,931 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,017 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-10T02:10:36,434 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:10:36,435 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,435 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,435 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,435 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,451 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,451 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,451 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,456 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,456 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,456 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:36,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:36,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:37,747 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:37,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:38,715 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 after 4001ms
2024-11-10T02:10:38,716 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/WALs/0c4d4d227d60,38109,1731204604181/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 to hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/oldWALs/0c4d4d227d60%2C38109%2C1731204604181.1731204604616
2024-11-10T02:10:38,718 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/MasterData/oldWALs/0c4d4d227d60%2C38109%2C1731204604181.1731204604616 to hdfs://localhost:41207/user/jenkins/test-data/511c9d22-23b1-ef78-6a99-82523e1ef5ed/oldWALs/0c4d4d227d60%2C38109%2C1731204604181.1731204604616$masterlocalwal$
2024-11-10T02:10:38,718 INFO  [M:0;0c4d4d227d60:38109 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:10:38,718 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:10:38,718 INFO  [M:0;0c4d4d227d60:38109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38109
2024-11-10T02:10:38,718 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:10:38,748 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:38,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:38,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:38,885 INFO  [M:0;0c4d4d227d60:38109 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:10:38,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38109-0x1012281dcad0000, quorum=127.0.0.1:59067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:10:38,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@574f69be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:38,888 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2da9d855{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:38,888 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:38,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27b0e5fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:38,888 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cbccde7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:38,889 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:38,889 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:38,889 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:38,889 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid 48ea7c40-b0f7-48ae-b452-7d5050a9edc0) service to localhost/127.0.0.1:41207
2024-11-10T02:10:38,890 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data3/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:38,890 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data4/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:38,890 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:38,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d867aab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:38,892 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a9d36bb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:38,892 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:38,892 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76a8c1ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:38,893 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c0d41d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:38,893 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:10:38,893 WARN  [BP-1556843191-172.17.0.3-1731204601934 heartbeating to localhost/127.0.0.1:41207 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1556843191-172.17.0.3-1731204601934 (Datanode Uuid da5fbcfd-db6f-4ab3-8a81-6a033ecb72f8) service to localhost/127.0.0.1:41207
2024-11-10T02:10:38,894 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data1/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:38,894 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/cluster_4f8a6103-713e-f4e7-d295-73f9866a6830/data/data2/current/BP-1556843191-172.17.0.3-1731204601934 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:10:38,894 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:10:38,894 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:10:38,895 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:10:38,900 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@650740c5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:10:38,900 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a99c341{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:10:38,900 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:10:38,900 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37ba1ac4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:10:38,900 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40c321ed{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir/,STOPPED}
2024-11-10T02:10:38,907 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:10:38,934 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:10:38,947 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 155)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41207
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41207
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41207 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41207
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41207 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:41207
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41207 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41207
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=294 (was 358), ProcessCount=11 (was 11), AvailableMemoryMB=3867 (was 4065)
2024-11-10T02:10:38,961 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=3867
2024-11-10T02:10:38,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.log.dir so I do NOT create it in target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3c9d9620-423c-39a6-1f7e-75b8e328a118/hadoop.tmp.dir so I do NOT create it in target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085, deleteOnExit=true
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/test.cache.data in system properties and HBase conf
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:10:38,962 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:10:38,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:10:38,963 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:10:38,964 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:10:38,976 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:10:39,307 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:39,311 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:39,312 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:39,312 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:39,312 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:39,312 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:39,312 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46a495b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:39,313 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cf5e3df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:39,408 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b5fa12b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/java.io.tmpdir/jetty-localhost-39635-hadoop-hdfs-3_4_1-tests_jar-_-any-11320765156028299071/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:10:39,408 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10128232{HTTP/1.1, (http/1.1)}{localhost:39635}
2024-11-10T02:10:39,408 INFO  [Time-limited test {}] server.Server(415): Started @196884ms
2024-11-10T02:10:39,421 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:10:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:10:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:10:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-10T02:10:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-10T02:10:39,647 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:39,651 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:39,652 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:39,652 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:39,652 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:39,652 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69ff6367{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:39,653 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4911f4c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:39,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:39,754 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10fa3dab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/java.io.tmpdir/jetty-localhost-45451-hadoop-hdfs-3_4_1-tests_jar-_-any-13261649679502043509/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:39,754 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@bc64a6a{HTTP/1.1, (http/1.1)}{localhost:45451}
2024-11-10T02:10:39,754 INFO  [Time-limited test {}] server.Server(415): Started @197230ms
2024-11-10T02:10:39,755 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:39,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:39,784 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:10:39,787 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:10:39,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:10:39,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:10:39,788 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:10:39,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dca8de9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:10:39,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6623a859{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:10:39,884 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e363d0c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/java.io.tmpdir/jetty-localhost-34603-hadoop-hdfs-3_4_1-tests_jar-_-any-2043224903029402649/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:10:39,884 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5daea1f2{HTTP/1.1, (http/1.1)}{localhost:34603}
2024-11-10T02:10:39,884 INFO  [Time-limited test {}] server.Server(415): Started @197360ms
2024-11-10T02:10:39,885 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:10:40,749 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:40,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:41,032 WARN  [Thread-1665 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data1/current/BP-1578170227-172.17.0.3-1731204638989/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:41,032 WARN  [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data2/current/BP-1578170227-172.17.0.3-1731204638989/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:41,050 WARN  [Thread-1629 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:41,052 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2cb07c74f52e854 with lease ID 0x34a84bf20a295240: Processing first storage report for DS-afc01cb6-4a92-4800-b3ac-2b3e7c40d6a5 from datanode DatanodeRegistration(127.0.0.1:39367, datanodeUuid=2de21a31-01be-4cfd-a656-351e3d0269dd, infoPort=44087, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989)
2024-11-10T02:10:41,052 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2cb07c74f52e854 with lease ID 0x34a84bf20a295240: from storage DS-afc01cb6-4a92-4800-b3ac-2b3e7c40d6a5 node DatanodeRegistration(127.0.0.1:39367, datanodeUuid=2de21a31-01be-4cfd-a656-351e3d0269dd, infoPort=44087, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-10T02:10:41,052 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2cb07c74f52e854 with lease ID 0x34a84bf20a295240: Processing first storage report for DS-ee442f10-6fc3-4538-a476-5b4289763599 from datanode DatanodeRegistration(127.0.0.1:39367, datanodeUuid=2de21a31-01be-4cfd-a656-351e3d0269dd, infoPort=44087, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989)
2024-11-10T02:10:41,052 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2cb07c74f52e854 with lease ID 0x34a84bf20a295240: from storage DS-ee442f10-6fc3-4538-a476-5b4289763599 node DatanodeRegistration(127.0.0.1:39367, datanodeUuid=2de21a31-01be-4cfd-a656-351e3d0269dd, infoPort=44087, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:41,192 WARN  [Thread-1677 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data4/current/BP-1578170227-172.17.0.3-1731204638989/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:41,192 WARN  [Thread-1676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data3/current/BP-1578170227-172.17.0.3-1731204638989/current, will proceed with Du for space computation calculation, 
2024-11-10T02:10:41,207 WARN  [Thread-1652 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:10:41,209 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8d19c08a2f186b8 with lease ID 0x34a84bf20a295241: Processing first storage report for DS-c9a03ea0-07e0-4b8d-9a2e-a838ffa514fe from datanode DatanodeRegistration(127.0.0.1:46399, datanodeUuid=eed11457-ad99-49fd-91bb-d4f750471e61, infoPort=41081, infoSecurePort=0, ipcPort=43483, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989)
2024-11-10T02:10:41,209 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8d19c08a2f186b8 with lease ID 0x34a84bf20a295241: from storage DS-c9a03ea0-07e0-4b8d-9a2e-a838ffa514fe node DatanodeRegistration(127.0.0.1:46399, datanodeUuid=eed11457-ad99-49fd-91bb-d4f750471e61, infoPort=41081, infoSecurePort=0, ipcPort=43483, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:41,209 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc8d19c08a2f186b8 with lease ID 0x34a84bf20a295241: Processing first storage report for DS-3e92e8be-c8ca-489c-836e-67a6ca3700fe from datanode DatanodeRegistration(127.0.0.1:46399, datanodeUuid=eed11457-ad99-49fd-91bb-d4f750471e61, infoPort=41081, infoSecurePort=0, ipcPort=43483, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989)
2024-11-10T02:10:41,209 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc8d19c08a2f186b8 with lease ID 0x34a84bf20a295241: from storage DS-3e92e8be-c8ca-489c-836e-67a6ca3700fe node DatanodeRegistration(127.0.0.1:46399, datanodeUuid=eed11457-ad99-49fd-91bb-d4f750471e61, infoPort=41081, infoSecurePort=0, ipcPort=43483, storageInfo=lv=-57;cid=testClusterID;nsid=1118139164;c=1731204638989), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:10:41,214 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d
2024-11-10T02:10:41,240 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/zookeeper_0, clientPort=52032, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:10:41,241 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52032
2024-11-10T02:10:41,241 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,243 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:10:41,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:10:41,253 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385 with version=8
2024-11-10T02:10:41,253 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:10:41,255 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:10:41,256 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33471
2024-11-10T02:10:41,257 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33471 connecting to ZooKeeper ensemble=127.0.0.1:52032
2024-11-10T02:10:41,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:334710x0, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:10:41,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33471-0x10122826d820000 connected
2024-11-10T02:10:41,396 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,397 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:41,399 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385, hbase.cluster.distributed=false
2024-11-10T02:10:41,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:10:41,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33471
2024-11-10T02:10:41,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33471
2024-11-10T02:10:41,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33471
2024-11-10T02:10:41,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33471
2024-11-10T02:10:41,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33471
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:10:41,417 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:10:41,418 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46471
2024-11-10T02:10:41,419 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46471 connecting to ZooKeeper ensemble=127.0.0.1:52032
2024-11-10T02:10:41,420 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,421 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464710x0, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:10:41,433 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464710x0, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:10:41,433 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46471-0x10122826d820001 connected
2024-11-10T02:10:41,434 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:10:41,434 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:10:41,435 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:10:41,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:10:41,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46471
2024-11-10T02:10:41,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46471
2024-11-10T02:10:41,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46471
2024-11-10T02:10:41,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46471
2024-11-10T02:10:41,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46471
2024-11-10T02:10:41,447 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:33471
2024-11-10T02:10:41,448 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:41,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:41,459 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:10:41,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,470 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:10:41,470 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,33471,1731204641254 from backup master directory
2024-11-10T02:10:41,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:41,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:10:41,480 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:10:41,480 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,484 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/hbase.id] with ID: 61fcfa51-4817-4d71-b757-abea85c85cdb
2024-11-10T02:10:41,484 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/.tmp/hbase.id
2024-11-10T02:10:41,492 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:10:41,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:10:41,493 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/.tmp/hbase.id]:[hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/hbase.id]
2024-11-10T02:10:41,505 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:41,505 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:10:41,507 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-10T02:10:41,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:10:41,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:10:41,524 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:10:41,525 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:10:41,525 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:41,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:10:41,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:10:41,533 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store
2024-11-10T02:10:41,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:10:41,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:10:41,545 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:41,545 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:10:41,545 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204641545Disabling compacts and flushes for region at 1731204641545Disabling writes for close at 1731204641545Writing region close event to WAL at 1731204641545Closed at 1731204641545
2024-11-10T02:10:41,546 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/.initializing
2024-11-10T02:10:41,546 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/WALs/0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,549 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C33471%2C1731204641254, suffix=, logDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/WALs/0c4d4d227d60,33471,1731204641254, archiveDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/oldWALs, maxLogs=10
2024-11-10T02:10:41,549 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C33471%2C1731204641254.1731204641549
2024-11-10T02:10:41,558 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/WALs/0c4d4d227d60,33471,1731204641254/0c4d4d227d60%2C33471%2C1731204641254.1731204641549
2024-11-10T02:10:41,564 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41081:41081),(127.0.0.1/127.0.0.1:44087:44087)]
2024-11-10T02:10:41,565 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:41,565 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:41,565 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,565 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,567 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:41,568 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:10:41,568 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:41,569 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:41,569 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:41,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:10:41,570 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:41,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:41,570 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:41,571 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:10:41,571 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:41,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:41,572 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:10:41,573 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:10:41,573 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:41,574 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:41,574 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,575 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,575 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,576 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,576 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,577 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:10:41,578 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:10:41,581 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:41,581 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=746211, jitterRate=-0.05114381015300751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:10:41,582 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204641566Initializing all the Stores at 1731204641566Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204641567 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204641567Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204641567Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204641567Cleaning up temporary data from old regions at 1731204641576 (+9 ms)Region opened successfully at 1731204641582 (+6 ms)
2024-11-10T02:10:41,582 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:10:41,585 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5344b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:10:41,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:10:41,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:10:41,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:10:41,586 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:10:41,587 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:10:41,587 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:10:41,587 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:10:41,590 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:10:41,591 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:10:41,601 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:10:41,602 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:10:41,602 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:10:41,612 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:10:41,612 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:10:41,613 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:10:41,622 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:10:41,623 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:10:41,633 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:10:41,635 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:10:41,643 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:10:41,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:41,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:10:41,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,654 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,33471,1731204641254, sessionid=0x10122826d820000, setting cluster-up flag (Was=false)
2024-11-10T02:10:41,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,707 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:10:41,708 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:41,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:41,759 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:10:41,760 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:41,762 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:10:41,763 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:41,764 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:10:41,764 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:10:41,764 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,33471,1731204641254
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:10:41,766 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,767 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204671767
2024-11-10T02:10:41,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:41,767 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,768 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:41,768 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:10:41,768 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:10:41,769 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:10:41,769 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204641769,5,FailOnTimeoutGroup]
2024-11-10T02:10:41,769 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204641769,5,FailOnTimeoutGroup]
2024-11-10T02:10:41,769 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,769 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:10:41,769 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,769 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,769 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:41,769 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:10:41,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:10:41,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:10:41,778 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:10:41,778 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385
2024-11-10T02:10:41,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:10:41,790 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:10:41,839 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(746): ClusterId : 61fcfa51-4817-4d71-b757-abea85c85cdb
2024-11-10T02:10:41,839 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:10:41,849 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:10:41,849 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:10:41,860 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:10:41,860 DEBUG [RS:0;0c4d4d227d60:46471 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1951763b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:10:41,878 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:46471
2024-11-10T02:10:41,878 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:10:41,878 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:10:41,878 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:10:41,879 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,33471,1731204641254 with port=46471, startcode=1731204641417
2024-11-10T02:10:41,879 DEBUG [RS:0;0c4d4d227d60:46471 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:10:41,882 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58575, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:10:41,882 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,883 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33471 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,885 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385
2024-11-10T02:10:41,885 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45733
2024-11-10T02:10:41,885 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:10:41,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:10:41,896 DEBUG [RS:0;0c4d4d227d60:46471 {}] zookeeper.ZKUtil(111): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,896 WARN  [RS:0;0c4d4d227d60:46471 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:10:41,896 INFO  [RS:0;0c4d4d227d60:46471 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:41,897 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,897 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,46471,1731204641417]
2024-11-10T02:10:41,900 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:10:41,901 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:10:41,903 INFO  [RS:0;0c4d4d227d60:46471 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:10:41,903 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,903 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:10:41,904 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:10:41,904 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:10:41,904 DEBUG [RS:0;0c4d4d227d60:46471 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,905 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46471,1731204641417-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:10:41,921 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:10:41,921 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,46471,1731204641417-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,921 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,921 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.Replication(171): 0c4d4d227d60,46471,1731204641417 started
2024-11-10T02:10:41,935 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:41,935 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,46471,1731204641417, RpcServer on 0c4d4d227d60/172.17.0.3:46471, sessionid=0x10122826d820001
2024-11-10T02:10:41,935 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:10:41,935 DEBUG [RS:0;0c4d4d227d60:46471 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,935 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,46471,1731204641417'
2024-11-10T02:10:41,935 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,46471,1731204641417'
2024-11-10T02:10:41,936 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:10:41,937 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:10:41,937 DEBUG [RS:0;0c4d4d227d60:46471 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:10:41,937 INFO  [RS:0;0c4d4d227d60:46471 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:10:41,937 INFO  [RS:0;0c4d4d227d60:46471 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:10:42,039 INFO  [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C46471%2C1731204641417, suffix=, logDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417, archiveDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs, maxLogs=32
2024-11-10T02:10:42,039 INFO  [RS:0;0c4d4d227d60:46471 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46471%2C1731204641417.1731204642039
2024-11-10T02:10:42,048 INFO  [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204642039
2024-11-10T02:10:42,049 DEBUG [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44087:44087),(127.0.0.1/127.0.0.1:41081:41081)]
2024-11-10T02:10:42,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:42,192 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:10:42,194 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:10:42,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,194 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,195 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:10:42,196 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:10:42,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,196 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,196 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:10:42,198 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:10:42,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,198 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,199 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:10:42,200 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:10:42,200 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,200 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:10:42,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740
2024-11-10T02:10:42,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740
2024-11-10T02:10:42,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:10:42,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:10:42,203 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:10:42,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:10:42,206 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:42,206 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859146, jitterRate=0.09246143698692322}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204642191Initializing all the Stores at 1731204642192 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642192Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642192Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204642192Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642192Cleaning up temporary data from old regions at 1731204642203 (+11 ms)Region opened successfully at 1731204642207 (+4 ms)
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:10:42,207 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:10:42,207 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:10:42,207 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204642207Disabling compacts and flushes for region at 1731204642207Disabling writes for close at 1731204642207Writing region close event to WAL at 1731204642207Closed at 1731204642207
2024-11-10T02:10:42,208 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:42,209 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:10:42,209 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:10:42,210 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:10:42,211 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:10:42,361 DEBUG [0c4d4d227d60:33471 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:10:42,362 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:42,363 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,46471,1731204641417, state=OPENING
2024-11-10T02:10:42,416 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:10:42,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:42,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:10:42,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:42,428 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:42,428 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:10:42,428 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,46471,1731204641417}]
2024-11-10T02:10:42,581 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:10:42,583 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43661, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:10:42,587 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:10:42,587 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:10:42,589 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C46471%2C1731204641417.meta, suffix=.meta, logDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417, archiveDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs, maxLogs=32
2024-11-10T02:10:42,589 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46471%2C1731204641417.meta.1731204642589.meta
2024-11-10T02:10:42,595 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.meta.1731204642589.meta
2024-11-10T02:10:42,599 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44087:44087),(127.0.0.1/127.0.0.1:41081:41081)]
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:10:42,608 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:42,608 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:10:42,609 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:10:42,610 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:10:42,611 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:10:42,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,611 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,611 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:10:42,612 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:10:42,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,613 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,613 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:10:42,613 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:10:42,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,614 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,614 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:10:42,614 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:10:42,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,615 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:10:42,615 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:10:42,616 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740
2024-11-10T02:10:42,617 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740
2024-11-10T02:10:42,618 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:10:42,618 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:10:42,618 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:10:42,620 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:10:42,620 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871770, jitterRate=0.10851407051086426}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:10:42,620 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:10:42,621 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204642609Writing region info on filesystem at 1731204642609Initializing all the Stores at 1731204642610 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642610Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642610Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204642610Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204642610Cleaning up temporary data from old regions at 1731204642618 (+8 ms)Running coprocessor post-open hooks at 1731204642620 (+2 ms)Region opened successfully at 1731204642621 (+1 ms)
2024-11-10T02:10:42,622 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204642581
2024-11-10T02:10:42,624 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:10:42,625 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:10:42,625 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:42,627 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,46471,1731204641417, state=OPEN
2024-11-10T02:10:42,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:10:42,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:10:42,725 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:42,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:42,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:10:42,728 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:10:42,729 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,46471,1731204641417 in 297 msec
2024-11-10T02:10:42,731 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:10:42,731 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 520 msec
2024-11-10T02:10:42,731 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:10:42,731 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:10:42,733 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:10:42,733 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,46471,1731204641417, seqNum=-1]
2024-11-10T02:10:42,733 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:10:42,734 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47681, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:10:42,740 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 976 msec
2024-11-10T02:10:42,740 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204642740, completionTime=-1
2024-11-10T02:10:42,740 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:10:42,740 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204702742
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204762742
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:33471, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,742 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,743 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,744 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.266sec
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:10:42,746 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:10:42,748 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:10:42,749 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:10:42,749 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,33471,1731204641254-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:10:42,750 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:42,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:42,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e442426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:42,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,33471,-1 for getting cluster id
2024-11-10T02:10:42,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:10:42,843 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '61fcfa51-4817-4d71-b757-abea85c85cdb'
2024-11-10T02:10:42,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:10:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "61fcfa51-4817-4d71-b757-abea85c85cdb"

2024-11-10T02:10:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb064cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,33471,-1]
2024-11-10T02:10:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:10:42,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:10:42,845 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:10:42,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a63a5a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:10:42,847 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:10:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,46471,1731204641417, seqNum=-1]
2024-11-10T02:10:42,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:10:42,850 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:10:42,852 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:42,853 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:10:42,856 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:10:42,856 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-10T02:10:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c4d4d227d60,33471,1731204641254
2024-11-10T02:10:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13640fbc
2024-11-10T02:10:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-10T02:10:42,858 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-10T02:10:42,859 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-10T02:10:42,859 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-10T02:10:42,859 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:10:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:10:42,862 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-10T02:10:42,862 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:42,862 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-10T02:10:42,863 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-10T02:10:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:10:42,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741835_1011 (size=405)
2024-11-10T02:10:42,869 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741835_1011 (size=405)
2024-11-10T02:10:42,871 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b88b4e8c06a5c69d87946dd28bcee9d, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385
2024-11-10T02:10:42,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741836_1012 (size=88)
2024-11-10T02:10:42,878 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741836_1012 (size=88)
2024-11-10T02:10:42,878 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:42,879 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 7b88b4e8c06a5c69d87946dd28bcee9d, disabling compactions & flushes
2024-11-10T02:10:42,879 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:42,879 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:42,879 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d. after waiting 0 ms
2024-11-10T02:10:42,879 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:42,879 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:42,879 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b88b4e8c06a5c69d87946dd28bcee9d:
Waiting for close lock at 1731204642879Disabling compacts and flushes for region at 1731204642879Disabling writes for close at 1731204642879Writing region close event to WAL at 1731204642879Closed at 1731204642879
2024-11-10T02:10:42,880 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-10T02:10:42,880 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731204642880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204642880"}]},"ts":"1731204642880"}
2024-11-10T02:10:42,882 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-10T02:10:42,884 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-10T02:10:42,884 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204642884"}]},"ts":"1731204642884"}
2024-11-10T02:10:42,886 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-10T02:10:42,886 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b88b4e8c06a5c69d87946dd28bcee9d, ASSIGN}]
2024-11-10T02:10:42,888 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b88b4e8c06a5c69d87946dd28bcee9d, ASSIGN
2024-11-10T02:10:42,888 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b88b4e8c06a5c69d87946dd28bcee9d, ASSIGN; state=OFFLINE, location=0c4d4d227d60,46471,1731204641417; forceNewPlan=false, retain=false
2024-11-10T02:10:43,039 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b88b4e8c06a5c69d87946dd28bcee9d, regionState=OPENING, regionLocation=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:43,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b88b4e8c06a5c69d87946dd28bcee9d, ASSIGN because future has completed
2024-11-10T02:10:43,042 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b88b4e8c06a5c69d87946dd28bcee9d, server=0c4d4d227d60,46471,1731204641417}]
2024-11-10T02:10:43,198 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:43,199 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b88b4e8c06a5c69d87946dd28bcee9d, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:10:43,199 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,199 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:10:43,199 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,199 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,200 INFO  [StoreOpener-7b88b4e8c06a5c69d87946dd28bcee9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7b88b4e8c06a5c69d87946dd28bcee9d 
2024-11-10T02:10:43,202 INFO  [StoreOpener-7b88b4e8c06a5c69d87946dd28bcee9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b88b4e8c06a5c69d87946dd28bcee9d columnFamilyName info
2024-11-10T02:10:43,202 DEBUG [StoreOpener-7b88b4e8c06a5c69d87946dd28bcee9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:10:43,202 INFO  [StoreOpener-7b88b4e8c06a5c69d87946dd28bcee9d-1 {}] regionserver.HStore(327): Store=7b88b4e8c06a5c69d87946dd28bcee9d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:10:43,202 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,203 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,203 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,204 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,204 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,205 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,207 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:10:43,207 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7b88b4e8c06a5c69d87946dd28bcee9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=777085, jitterRate=-0.011885985732078552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:10:43,208 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:10:43,208 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7b88b4e8c06a5c69d87946dd28bcee9d:
Running coprocessor pre-open hook at 1731204643199Writing region info on filesystem at 1731204643199Initializing all the Stores at 1731204643200 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204643200Cleaning up temporary data from old regions at 1731204643204 (+4 ms)Running coprocessor post-open hooks at 1731204643208 (+4 ms)Region opened successfully at 1731204643208
2024-11-10T02:10:43,209 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d., pid=6, masterSystemTime=1731204643195
2024-11-10T02:10:43,212 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:43,212 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:43,213 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7b88b4e8c06a5c69d87946dd28bcee9d, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,46471,1731204641417
2024-11-10T02:10:43,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b88b4e8c06a5c69d87946dd28bcee9d, server=0c4d4d227d60,46471,1731204641417 because future has completed
2024-11-10T02:10:43,219 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-10T02:10:43,219 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b88b4e8c06a5c69d87946dd28bcee9d, server=0c4d4d227d60,46471,1731204641417 in 175 msec
2024-11-10T02:10:43,222 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-10T02:10:43,222 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=7b88b4e8c06a5c69d87946dd28bcee9d, ASSIGN in 333 msec
2024-11-10T02:10:43,223 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-10T02:10:43,223 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204643223"}]},"ts":"1731204643223"}
2024-11-10T02:10:43,225 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-10T02:10:43,227 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-10T02:10:43,229 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 368 msec
2024-11-10T02:10:43,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:43,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:44,751 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:44,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:44,924 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:10:44,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,940 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,941 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,942 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,945 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,946 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,946 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:44,949 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:10:45,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:45,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:46,752 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:46,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:47,753 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:47,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:47,900 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-10T02:10:47,900 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-10T02:10:48,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:48,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:49,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:10:49,422 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-10T02:10:49,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:10:49,422 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-10T02:10:49,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-10T02:10:49,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-10T02:10:49,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:10:49,423 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-10T02:10:49,754 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:49,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:50,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:50,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:51,755 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:51,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:52,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:52,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:10:52,941 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-10T02:10:52,941 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-10T02:10:52,944 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:10:52,944 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d., hostname=0c4d4d227d60,46471,1731204641417, seqNum=2]
2024-11-10T02:10:52,955 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:10:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:10:52,960 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-10T02:10:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-10T02:10:52,962 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-10T02:10:52,963 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-10T02:10:53,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-10T02:10:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:53,125 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 7b88b4e8c06a5c69d87946dd28bcee9d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:10:53,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/7c060deadb6f4d188e4db0ae6906e475 is 1080, key is row0001/info:/1731204652949/Put/seqid=0
2024-11-10T02:10:53,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741837_1013 (size=6033)
2024-11-10T02:10:53,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741837_1013 (size=6033)
2024-11-10T02:10:53,552 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/7c060deadb6f4d188e4db0ae6906e475
2024-11-10T02:10:53,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/7c060deadb6f4d188e4db0ae6906e475 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475
2024-11-10T02:10:53,565 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475, entries=1, sequenceid=5, filesize=5.9 K
2024-11-10T02:10:53,566 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 441ms, sequenceid=5, compaction requested=false
2024-11-10T02:10:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 7b88b4e8c06a5c69d87946dd28bcee9d:

2024-11-10T02:10:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:10:53,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-10T02:10:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-10T02:10:53,574 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-10T02:10:53,574 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 608 msec
2024-11-10T02:10:53,576 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 619 msec
2024-11-10T02:10:53,756 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:53,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:54,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:54,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:55,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:55,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:56,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:56,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:57,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:57,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:58,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:58,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:59,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:10:59,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:00,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:00,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:01,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:01,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:02,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:02,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996 after 68050ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:11:02,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:02,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta after 68045ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-10T02:11:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-10T02:11:03,061 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-10T02:11:03,064 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-10T02:11:03,066 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-10T02:11:03,067 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-10T02:11:03,068 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-10T02:11:03,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-10T02:11:03,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:03,222 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 7b88b4e8c06a5c69d87946dd28bcee9d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:11:03,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/f0c94f4fcc1e4d02b0f16c3f721d3fce is 1080, key is row0002/info:/1731204663062/Put/seqid=0
2024-11-10T02:11:03,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741838_1014 (size=6033)
2024-11-10T02:11:03,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741838_1014 (size=6033)
2024-11-10T02:11:03,236 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/f0c94f4fcc1e4d02b0f16c3f721d3fce
2024-11-10T02:11:03,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/f0c94f4fcc1e4d02b0f16c3f721d3fce as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce
2024-11-10T02:11:03,250 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce, entries=1, sequenceid=9, filesize=5.9 K
2024-11-10T02:11:03,251 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 30ms, sequenceid=9, compaction requested=false
2024-11-10T02:11:03,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 7b88b4e8c06a5c69d87946dd28bcee9d:

2024-11-10T02:11:03,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:03,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-10T02:11:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-10T02:11:03,255 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-10T02:11:03,255 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 186 msec
2024-11-10T02:11:03,258 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 192 msec
2024-11-10T02:11:03,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:03,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:04,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:04,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:05,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:05,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:06,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:06,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:07,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:07,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:08,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:08,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:09,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:09,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:10,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:10,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:11,214 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:11:11,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:11,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:12,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:12,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:13,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-10T02:11:13,171 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-10T02:11:13,174 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46471%2C1731204641417.1731204673173
2024-11-10T02:11:13,179 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:13,179 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:13,179 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:13,179 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:13,179 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:13,180 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204642039 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204673173
2024-11-10T02:11:13,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44087:44087),(127.0.0.1/127.0.0.1:41081:41081)]
2024-11-10T02:11:13,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204642039 is not closed yet, will try archiving it next time
2024-11-10T02:11:13,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741833_1009 (size=5546)
2024-11-10T02:11:13,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741833_1009 (size=5546)
2024-11-10T02:11:13,182 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-10T02:11:13,184 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-10T02:11:13,185 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-10T02:11:13,185 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-10T02:11:13,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46471 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-10T02:11:13,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:13,338 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 7b88b4e8c06a5c69d87946dd28bcee9d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:11:13,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/cc102fd220f141b5b86da0b948d40b99 is 1080, key is row0003/info:/1731204673172/Put/seqid=0
2024-11-10T02:11:13,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741840_1016 (size=6033)
2024-11-10T02:11:13,347 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/cc102fd220f141b5b86da0b948d40b99
2024-11-10T02:11:13,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741840_1016 (size=6033)
2024-11-10T02:11:13,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/cc102fd220f141b5b86da0b948d40b99 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99
2024-11-10T02:11:13,361 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99, entries=1, sequenceid=13, filesize=5.9 K
2024-11-10T02:11:13,362 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 24ms, sequenceid=13, compaction requested=true
2024-11-10T02:11:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 7b88b4e8c06a5c69d87946dd28bcee9d:

2024-11-10T02:11:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:13,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-10T02:11:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-10T02:11:13,367 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-10T02:11:13,367 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec
2024-11-10T02:11:13,370 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec
2024-11-10T02:11:13,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:13,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:14,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:14,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:15,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:15,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:16,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:16,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:17,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:17,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:18,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:18,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:19,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:19,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:20,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:20,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:21,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:21,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:22,751 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-10T02:11:22,751 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-10T02:11:22,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:22,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-10T02:11:23,241 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-10T02:11:23,241 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:11:23,242 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:11:23,242 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7b88b4e8c06a5c69d87946dd28bcee9d/info is initiating minor compaction (all files)
2024-11-10T02:11:23,242 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:11:23,242 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:23,242 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7b88b4e8c06a5c69d87946dd28bcee9d/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:23,242 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99] into tmpdir=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp, totalSize=17.7 K
2024-11-10T02:11:23,243 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7c060deadb6f4d188e4db0ae6906e475, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731204652949
2024-11-10T02:11:23,243 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f0c94f4fcc1e4d02b0f16c3f721d3fce, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731204663062
2024-11-10T02:11:23,244 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cc102fd220f141b5b86da0b948d40b99, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731204673172
2024-11-10T02:11:23,254 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 7b88b4e8c06a5c69d87946dd28bcee9d#info#compaction#47 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:23,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/f986f01e00d4403e878179003235dba3 is 1080, key is row0001/info:/1731204652949/Put/seqid=0
2024-11-10T02:11:23,259 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741841_1017 (size=8296)
2024-11-10T02:11:23,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741841_1017 (size=8296)
2024-11-10T02:11:23,265 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/f986f01e00d4403e878179003235dba3 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f986f01e00d4403e878179003235dba3
2024-11-10T02:11:23,271 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7b88b4e8c06a5c69d87946dd28bcee9d/info of 7b88b4e8c06a5c69d87946dd28bcee9d into f986f01e00d4403e878179003235dba3(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:23,271 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7b88b4e8c06a5c69d87946dd28bcee9d:

2024-11-10T02:11:23,274 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46471%2C1731204641417.1731204683274
2024-11-10T02:11:23,279 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:23,279 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:23,279 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:23,279 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:23,279 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:23,280 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204673173 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204683274
2024-11-10T02:11:23,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41081:41081),(127.0.0.1/127.0.0.1:44087:44087)]
2024-11-10T02:11:23,280 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204673173 is not closed yet, will try archiving it next time
2024-11-10T02:11:23,281 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204642039 to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs/0c4d4d227d60%2C46471%2C1731204641417.1731204642039
2024-11-10T02:11:23,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741839_1015 (size=2520)
2024-11-10T02:11:23,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741839_1015 (size=2520)
2024-11-10T02:11:23,281 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-10T02:11:23,284 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-10T02:11:23,284 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-10T02:11:23,284 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-10T02:11:23,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46471 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-10T02:11:23,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:23,438 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 7b88b4e8c06a5c69d87946dd28bcee9d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:11:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/5e1bc38e6c934d8bb44113193bb9095d is 1080, key is row0000/info:/1731204683272/Put/seqid=0
2024-11-10T02:11:23,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741843_1019 (size=6033)
2024-11-10T02:11:23,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741843_1019 (size=6033)
2024-11-10T02:11:23,447 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/5e1bc38e6c934d8bb44113193bb9095d
2024-11-10T02:11:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/5e1bc38e6c934d8bb44113193bb9095d as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/5e1bc38e6c934d8bb44113193bb9095d
2024-11-10T02:11:23,458 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/5e1bc38e6c934d8bb44113193bb9095d, entries=1, sequenceid=18, filesize=5.9 K
2024-11-10T02:11:23,459 INFO  [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 22ms, sequenceid=18, compaction requested=false
2024-11-10T02:11:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 7b88b4e8c06a5c69d87946dd28bcee9d:

2024-11-10T02:11:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-10T02:11:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-10T02:11:23,464 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-10T02:11:23,464 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-11-10T02:11:23,466 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-11-10T02:11:23,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:23,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:24,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:24,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:25,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:25,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:26,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:26,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:27,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:27,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:28,199 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 7b88b4e8c06a5c69d87946dd28bcee9d, had cached 0 bytes from a total of 14329
2024-11-10T02:11:28,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:28,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:29,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:29,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:30,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:30,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:31,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:31,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:32,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:32,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:33,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-10T02:11:33,341 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-10T02:11:33,344 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C46471%2C1731204641417.1731204693343
2024-11-10T02:11:33,349 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,349 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,349 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,349 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,349 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,349 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204683274 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204693343
2024-11-10T02:11:33,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44087:44087),(127.0.0.1/127.0.0.1:41081:41081)]
2024-11-10T02:11:33,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204683274 is not closed yet, will try archiving it next time
2024-11-10T02:11:33,350 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/WALs/0c4d4d227d60,46471,1731204641417/0c4d4d227d60%2C46471%2C1731204641417.1731204673173 to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs/0c4d4d227d60%2C46471%2C1731204641417.1731204673173
2024-11-10T02:11:33,350 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:11:33,350 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:11:33,351 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:11:33,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:33,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:33,351 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:11:33,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741842_1018 (size=2026)
2024-11-10T02:11:33,351 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945418508, stopped=false
2024-11-10T02:11:33,351 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,33471,1731204641254
2024-11-10T02:11:33,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741842_1018 (size=2026)
2024-11-10T02:11:33,351 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:11:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:11:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:11:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:33,414 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:11:33,414 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:11:33,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:11:33,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:33,415 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,46471,1731204641417' *****
2024-11-10T02:11:33,415 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:11:33,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:11:33,415 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,46471,1731204641417
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:11:33,415 INFO  [RS:0;0c4d4d227d60:46471 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:46471.
2024-11-10T02:11:33,415 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b88b4e8c06a5c69d87946dd28bcee9d, disabling compactions & flushes
2024-11-10T02:11:33,416 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:33,416 DEBUG [RS:0;0c4d4d227d60:46471 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:33,416 DEBUG [RS:0;0c4d4d227d60:46471 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d. after waiting 0 ms
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:33,416 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:11:33,416 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:11:33,416 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:11:33,416 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7b88b4e8c06a5c69d87946dd28bcee9d 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:11:33,416 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:11:33,416 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-10T02:11:33,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:11:33,416 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1325): Online Regions={7b88b4e8c06a5c69d87946dd28bcee9d=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d., 1588230740=hbase:meta,,1.1588230740}
2024-11-10T02:11:33,416 DEBUG [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7b88b4e8c06a5c69d87946dd28bcee9d
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:11:33,416 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:11:33,416 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:11:33,416 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-10T02:11:33,420 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/1114e0415a7841fca63a77034c83642d is 1080, key is row0001/info:/1731204693342/Put/seqid=0
2024-11-10T02:11:33,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741845_1021 (size=6033)
2024-11-10T02:11:33,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741845_1021 (size=6033)
2024-11-10T02:11:33,427 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/1114e0415a7841fca63a77034c83642d
2024-11-10T02:11:33,433 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/info/44a1218ae3ba464da50c2eb1a50b50da is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d./info:regioninfo/1731204643213/Put/seqid=0
2024-11-10T02:11:33,434 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/.tmp/info/1114e0415a7841fca63a77034c83642d as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/1114e0415a7841fca63a77034c83642d
2024-11-10T02:11:33,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741846_1022 (size=7308)
2024-11-10T02:11:33,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741846_1022 (size=7308)
2024-11-10T02:11:33,438 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/info/44a1218ae3ba464da50c2eb1a50b50da
2024-11-10T02:11:33,440 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/1114e0415a7841fca63a77034c83642d, entries=1, sequenceid=22, filesize=5.9 K
2024-11-10T02:11:33,441 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 25ms, sequenceid=22, compaction requested=true
2024-11-10T02:11:33,441 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99] to archive
2024-11-10T02:11:33,442 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:11:33,443 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475 to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/7c060deadb6f4d188e4db0ae6906e475
2024-11-10T02:11:33,445 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/f0c94f4fcc1e4d02b0f16c3f721d3fce
2024-11-10T02:11:33,446 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99 to hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/info/cc102fd220f141b5b86da0b948d40b99
2024-11-10T02:11:33,447 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0c4d4d227d60:33471 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-10T02:11:33,447 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7c060deadb6f4d188e4db0ae6906e475=6033, f0c94f4fcc1e4d02b0f16c3f721d3fce=6033, cc102fd220f141b5b86da0b948d40b99=6033]
2024-11-10T02:11:33,451 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/7b88b4e8c06a5c69d87946dd28bcee9d/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-10T02:11:33,451 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:33,451 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b88b4e8c06a5c69d87946dd28bcee9d:
Waiting for close lock at 1731204693415Running coprocessor pre-close hooks at 1731204693415Disabling compacts and flushes for region at 1731204693415Disabling writes for close at 1731204693416 (+1 ms)Obtaining lock to block concurrent updates at 1731204693416Preparing flush snapshotting stores in 7b88b4e8c06a5c69d87946dd28bcee9d at 1731204693416Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731204693416Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d. at 1731204693417 (+1 ms)Flushing 7b88b4e8c06a5c69d87946dd28bcee9d/info: creating writer at 1731204693417Flushing 7b88b4e8c06a5c69d87946dd28bcee9d/info: appending metadata at 1731204693419 (+2 ms)Flushing 7b88b4e8c06a5c69d87946dd28bcee9d/info: closing flushed file at 1731204693419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d06be81: reopening flushed file at 1731204693433 (+14 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 7b88b4e8c06a5c69d87946dd28bcee9d in 25ms, sequenceid=22, compaction requested=true at 1731204693441 (+8 ms)Writing region close event to WAL at 1731204693447 (+6 ms)Running coprocessor post-close hooks at 1731204693451 (+4 ms)Closed at 1731204693451
2024-11-10T02:11:33,451 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731204642859.7b88b4e8c06a5c69d87946dd28bcee9d.
2024-11-10T02:11:33,458 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/ns/0911df0b90624623a4b51db3aad2b0bc is 43, key is default/ns:d/1731204642735/Put/seqid=0
2024-11-10T02:11:33,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741847_1023 (size=5153)
2024-11-10T02:11:33,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741847_1023 (size=5153)
2024-11-10T02:11:33,463 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/ns/0911df0b90624623a4b51db3aad2b0bc
2024-11-10T02:11:33,483 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/table/f5e6e19f261347f99299e1958974ba64 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731204643223/Put/seqid=0
2024-11-10T02:11:33,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741848_1024 (size=5508)
2024-11-10T02:11:33,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741848_1024 (size=5508)
2024-11-10T02:11:33,488 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/table/f5e6e19f261347f99299e1958974ba64
2024-11-10T02:11:33,495 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/info/44a1218ae3ba464da50c2eb1a50b50da as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/info/44a1218ae3ba464da50c2eb1a50b50da
2024-11-10T02:11:33,501 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/info/44a1218ae3ba464da50c2eb1a50b50da, entries=10, sequenceid=11, filesize=7.1 K
2024-11-10T02:11:33,502 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/ns/0911df0b90624623a4b51db3aad2b0bc as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/ns/0911df0b90624623a4b51db3aad2b0bc
2024-11-10T02:11:33,508 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/ns/0911df0b90624623a4b51db3aad2b0bc, entries=2, sequenceid=11, filesize=5.0 K
2024-11-10T02:11:33,509 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/.tmp/table/f5e6e19f261347f99299e1958974ba64 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/table/f5e6e19f261347f99299e1958974ba64
2024-11-10T02:11:33,515 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/table/f5e6e19f261347f99299e1958974ba64, entries=2, sequenceid=11, filesize=5.4 K
2024-11-10T02:11:33,516 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=11, compaction requested=false
2024-11-10T02:11:33,521 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-10T02:11:33,522 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:11:33,522 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:11:33,522 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204693416Running coprocessor pre-close hooks at 1731204693416Disabling compacts and flushes for region at 1731204693416Disabling writes for close at 1731204693416Obtaining lock to block concurrent updates at 1731204693416Preparing flush snapshotting stores in 1588230740 at 1731204693416Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731204693417 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731204693417Flushing 1588230740/info: creating writer at 1731204693417Flushing 1588230740/info: appending metadata at 1731204693432 (+15 ms)Flushing 1588230740/info: closing flushed file at 1731204693432Flushing 1588230740/ns: creating writer at 1731204693444 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731204693458 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731204693458Flushing 1588230740/table: creating writer at 1731204693469 (+11 ms)Flushing 1588230740/table: appending metadata at 1731204693483 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731204693483Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a1a63c4: reopening flushed file at 1731204693494 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8677ff: reopening flushed file at 1731204693501 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f7f937d: reopening flushed file at 1731204693508 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=11, compaction requested=false at 1731204693516 (+8 ms)Writing region close event to WAL at 1731204693517 (+1 ms)Running coprocessor post-close hooks at 1731204693522 (+5 ms)Closed at 1731204693522
2024-11-10T02:11:33,522 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:11:33,616 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,46471,1731204641417; all regions closed.
2024-11-10T02:11:33,617 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,617 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,617 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,617 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,617 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,619 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741834_1010 (size=3306)
2024-11-10T02:11:33,619 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741834_1010 (size=3306)
2024-11-10T02:11:33,621 DEBUG [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs
2024-11-10T02:11:33,621 INFO  [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C46471%2C1731204641417.meta:.meta(num 1731204642589)
2024-11-10T02:11:33,622 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,622 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,622 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,622 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,622 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741844_1020 (size=1252)
2024-11-10T02:11:33,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741844_1020 (size=1252)
2024-11-10T02:11:33,627 DEBUG [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/oldWALs
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C46471%2C1731204641417:(num 1731204693343)
2024-11-10T02:11:33,627 DEBUG [RS:0;0c4d4d227d60:46471 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:11:33,627 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:11:33,627 INFO  [RS:0;0c4d4d227d60:46471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46471
2024-11-10T02:11:33,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:11:33,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,46471,1731204641417
2024-11-10T02:11:33,639 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:11:33,650 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,46471,1731204641417]
2024-11-10T02:11:33,660 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,46471,1731204641417 already deleted, retry=false
2024-11-10T02:11:33,660 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,46471,1731204641417 expired; onlineServers=0
2024-11-10T02:11:33,660 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,33471,1731204641254' *****
2024-11-10T02:11:33,660 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:11:33,660 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:11:33,661 DEBUG [M:0;0c4d4d227d60:33471 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:11:33,661 DEBUG [M:0;0c4d4d227d60:33471 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:11:33,661 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:11:33,661 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204641769 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204641769,5,FailOnTimeoutGroup]
2024-11-10T02:11:33,661 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204641769 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204641769,5,FailOnTimeoutGroup]
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:11:33,661 DEBUG [M:0;0c4d4d227d60:33471 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:11:33,661 INFO  [M:0;0c4d4d227d60:33471 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:11:33,661 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:11:33,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:11:33,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:33,671 DEBUG [M:0;0c4d4d227d60:33471 {}] zookeeper.ZKUtil(347): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:11:33,671 WARN  [M:0;0c4d4d227d60:33471 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:11:33,672 INFO  [M:0;0c4d4d227d60:33471 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/.lastflushedseqids
2024-11-10T02:11:33,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741849_1025 (size=130)
2024-11-10T02:11:33,677 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741849_1025 (size=130)
2024-11-10T02:11:33,677 INFO  [M:0;0c4d4d227d60:33471 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:11:33,677 INFO  [M:0;0c4d4d227d60:33471 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:11:33,677 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:11:33,677 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:33,677 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:33,677 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:11:33,677 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:33,677 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.62 KB heapSize=55.02 KB
2024-11-10T02:11:33,691 DEBUG [M:0;0c4d4d227d60:33471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/859e998997ba420ebc64e8934270782a is 82, key is hbase:meta,,1/info:regioninfo/1731204642625/Put/seqid=0
2024-11-10T02:11:33,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741850_1026 (size=5672)
2024-11-10T02:11:33,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741850_1026 (size=5672)
2024-11-10T02:11:33,696 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/859e998997ba420ebc64e8934270782a
2024-11-10T02:11:33,714 DEBUG [M:0;0c4d4d227d60:33471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae619a6e410490cb113a889f1bb5790 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731204643228/Put/seqid=0
2024-11-10T02:11:33,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741851_1027 (size=7826)
2024-11-10T02:11:33,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741851_1027 (size=7826)
2024-11-10T02:11:33,719 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae619a6e410490cb113a889f1bb5790
2024-11-10T02:11:33,723 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bae619a6e410490cb113a889f1bb5790
2024-11-10T02:11:33,736 DEBUG [M:0;0c4d4d227d60:33471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad3ad35212ca441abe916516af3ab792 is 69, key is 0c4d4d227d60,46471,1731204641417/rs:state/1731204641883/Put/seqid=0
2024-11-10T02:11:33,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741852_1028 (size=5156)
2024-11-10T02:11:33,741 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741852_1028 (size=5156)
2024-11-10T02:11:33,741 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad3ad35212ca441abe916516af3ab792
2024-11-10T02:11:33,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:11:33,750 INFO  [RS:0;0c4d4d227d60:46471 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:11:33,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46471-0x10122826d820001, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:11:33,750 INFO  [RS:0;0c4d4d227d60:46471 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,46471,1731204641417; zookeeper connection closed.
2024-11-10T02:11:33,750 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e76011f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e76011f
2024-11-10T02:11:33,751 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:11:33,758 DEBUG [M:0;0c4d4d227d60:33471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c93172b89adb4eb5be8380a9db9f2710 is 52, key is load_balancer_on/state:d/1731204642854/Put/seqid=0
2024-11-10T02:11:33,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741853_1029 (size=5056)
2024-11-10T02:11:33,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741853_1029 (size=5056)
2024-11-10T02:11:33,764 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c93172b89adb4eb5be8380a9db9f2710
2024-11-10T02:11:33,769 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/859e998997ba420ebc64e8934270782a as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/859e998997ba420ebc64e8934270782a
2024-11-10T02:11:33,773 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/859e998997ba420ebc64e8934270782a, entries=8, sequenceid=121, filesize=5.5 K
2024-11-10T02:11:33,774 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bae619a6e410490cb113a889f1bb5790 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bae619a6e410490cb113a889f1bb5790
2024-11-10T02:11:33,779 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bae619a6e410490cb113a889f1bb5790
2024-11-10T02:11:33,779 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bae619a6e410490cb113a889f1bb5790, entries=14, sequenceid=121, filesize=7.6 K
2024-11-10T02:11:33,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:33,780 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ad3ad35212ca441abe916516af3ab792 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ad3ad35212ca441abe916516af3ab792
2024-11-10T02:11:33,785 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ad3ad35212ca441abe916516af3ab792, entries=1, sequenceid=121, filesize=5.0 K
2024-11-10T02:11:33,786 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c93172b89adb4eb5be8380a9db9f2710 as hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c93172b89adb4eb5be8380a9db9f2710
2024-11-10T02:11:33,790 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45733/user/jenkins/test-data/122b47dc-94ca-244a-b6de-ab58b2d90385/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c93172b89adb4eb5be8380a9db9f2710, entries=1, sequenceid=121, filesize=4.9 K
2024-11-10T02:11:33,791 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.62 KB/44668, heapSize ~54.96 KB/56280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=121, compaction requested=false
2024-11-10T02:11:33,793 INFO  [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:33,793 DEBUG [M:0;0c4d4d227d60:33471 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204693677Disabling compacts and flushes for region at 1731204693677Disabling writes for close at 1731204693677Obtaining lock to block concurrent updates at 1731204693677Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204693677Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44668, getHeapSize=56280, getOffHeapSize=0, getCellsCount=140 at 1731204693678 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204693678Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204693678Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204693691 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204693691Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204693700 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204693714 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204693714Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204693723 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204693736 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204693736Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204693745 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204693758 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204693758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc840c5: reopening flushed file at 1731204693768 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b4d27c: reopening flushed file at 1731204693773 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c28ade: reopening flushed file at 1731204693779 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fd90cb8: reopening flushed file at 1731204693785 (+6 ms)Finished flush of dataSize ~43.62 KB/44668, heapSize ~54.96 KB/56280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=121, compaction requested=false at 1731204693791 (+6 ms)Writing region close event to WAL at 1731204693793 (+2 ms)Closed at 1731204693793
2024-11-10T02:11:33,793 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,793 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,793 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,794 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,794 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:11:33,795 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39367 is added to blk_1073741830_1006 (size=53065)
2024-11-10T02:11:33,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46399 is added to blk_1073741830_1006 (size=53065)
2024-11-10T02:11:33,796 INFO  [M:0;0c4d4d227d60:33471 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:11:33,796 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:11:33,796 INFO  [M:0;0c4d4d227d60:33471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33471
2024-11-10T02:11:33,796 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:11:33,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:33,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:11:33,908 INFO  [M:0;0c4d4d227d60:33471 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:11:33,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33471-0x10122826d820000, quorum=127.0.0.1:52032, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:11:33,908 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:11:33,911 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e363d0c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:11:33,911 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5daea1f2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:11:33,911 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:11:33,911 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6623a859{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:11:33,911 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dca8de9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,STOPPED}
2024-11-10T02:11:33,913 WARN  [BP-1578170227-172.17.0.3-1731204638989 heartbeating to localhost/127.0.0.1:45733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:11:33,913 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:11:33,913 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:11:33,913 WARN  [BP-1578170227-172.17.0.3-1731204638989 heartbeating to localhost/127.0.0.1:45733 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1578170227-172.17.0.3-1731204638989 (Datanode Uuid eed11457-ad99-49fd-91bb-d4f750471e61) service to localhost/127.0.0.1:45733
2024-11-10T02:11:33,914 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data3/current/BP-1578170227-172.17.0.3-1731204638989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:11:33,914 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data4/current/BP-1578170227-172.17.0.3-1731204638989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:11:33,914 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:11:33,921 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10fa3dab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:11:33,921 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@bc64a6a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:11:33,921 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:11:33,921 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4911f4c5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:11:33,921 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69ff6367{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,STOPPED}
2024-11-10T02:11:33,923 WARN  [BP-1578170227-172.17.0.3-1731204638989 heartbeating to localhost/127.0.0.1:45733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:11:33,923 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:11:33,923 WARN  [BP-1578170227-172.17.0.3-1731204638989 heartbeating to localhost/127.0.0.1:45733 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1578170227-172.17.0.3-1731204638989 (Datanode Uuid 2de21a31-01be-4cfd-a656-351e3d0269dd) service to localhost/127.0.0.1:45733
2024-11-10T02:11:33,923 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:11:33,924 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data1/current/BP-1578170227-172.17.0.3-1731204638989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:11:33,924 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/cluster_a37a7734-02a9-af2c-57b1-48a6dff4b085/data/data2/current/BP-1578170227-172.17.0.3-1731204638989 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:11:33,924 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:11:33,930 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b5fa12b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:11:33,930 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10128232{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:11:33,930 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:11:33,931 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cf5e3df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:11:33,931 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46a495b0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir/,STOPPED}
2024-11-10T02:11:33,938 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:11:33,965 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:11:33,973 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 180)
Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45733
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45733 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45733 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45733
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:45733
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45733 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45733
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45733
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 294), ProcessCount=11 (was 11), AvailableMemoryMB=2825 (was 3867)
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=2821
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.log.dir so I do NOT create it in target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/19e41dfd-0c36-eb90-4f8d-e3c5fbb11c7d/hadoop.tmp.dir so I do NOT create it in target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360, deleteOnExit=true
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:11:33,982 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/test.cache.data in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:11:33,983 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:11:33,983 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:11:33,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:11:33,985 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:11:34,001 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:11:34,766 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:11:34,770 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:11:34,771 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:11:34,772 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:11:34,772 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:11:34,772 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:11:34,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7193a060{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:11:34,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f1304aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:11:34,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:34,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:34,866 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b6783f5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/java.io.tmpdir/jetty-localhost-35353-hadoop-hdfs-3_4_1-tests_jar-_-any-12090377318967643289/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:11:34,867 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44a25975{HTTP/1.1, (http/1.1)}{localhost:35353}
2024-11-10T02:11:34,867 INFO  [Time-limited test {}] server.Server(415): Started @252343ms
2024-11-10T02:11:34,879 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:11:35,162 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:11:35,164 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:11:35,165 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:11:35,165 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:11:35,165 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-10T02:11:35,166 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23e1642c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:11:35,166 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@601b78f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:11:35,261 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64358886{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/java.io.tmpdir/jetty-localhost-35269-hadoop-hdfs-3_4_1-tests_jar-_-any-6483265704509853023/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:11:35,262 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@26a9d62d{HTTP/1.1, (http/1.1)}{localhost:35269}
2024-11-10T02:11:35,262 INFO  [Time-limited test {}] server.Server(415): Started @252738ms
2024-11-10T02:11:35,263 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:11:35,291 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:11:35,293 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:11:35,294 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:11:35,294 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:11:35,294 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:11:35,294 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47bcda8c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:11:35,295 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f4aa33e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:11:35,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@20408ea5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/java.io.tmpdir/jetty-localhost-42965-hadoop-hdfs-3_4_1-tests_jar-_-any-15633759816105533417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:11:35,395 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e2ab763{HTTP/1.1, (http/1.1)}{localhost:42965}
2024-11-10T02:11:35,395 INFO  [Time-limited test {}] server.Server(415): Started @252871ms
2024-11-10T02:11:35,396 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:11:35,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:35,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:36,579 WARN  [Thread-1981 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data1/current/BP-292189752-172.17.0.3-1731204694004/current, will proceed with Du for space computation calculation, 
2024-11-10T02:11:36,580 WARN  [Thread-1982 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data2/current/BP-292189752-172.17.0.3-1731204694004/current, will proceed with Du for space computation calculation, 
2024-11-10T02:11:36,594 WARN  [Thread-1945 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:11:36,596 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ca18b01a9cd5a63 with lease ID 0xd58ebbc5f537379b: Processing first storage report for DS-71a71186-1e9d-41a1-b679-f1184511622f from datanode DatanodeRegistration(127.0.0.1:41963, datanodeUuid=a25e57ba-dbec-409c-92eb-94aa44af277f, infoPort=33623, infoSecurePort=0, ipcPort=34195, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004)
2024-11-10T02:11:36,596 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ca18b01a9cd5a63 with lease ID 0xd58ebbc5f537379b: from storage DS-71a71186-1e9d-41a1-b679-f1184511622f node DatanodeRegistration(127.0.0.1:41963, datanodeUuid=a25e57ba-dbec-409c-92eb-94aa44af277f, infoPort=33623, infoSecurePort=0, ipcPort=34195, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:11:36,596 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ca18b01a9cd5a63 with lease ID 0xd58ebbc5f537379b: Processing first storage report for DS-c8b96d83-36b4-4300-bd7e-f5936e3ed9f3 from datanode DatanodeRegistration(127.0.0.1:41963, datanodeUuid=a25e57ba-dbec-409c-92eb-94aa44af277f, infoPort=33623, infoSecurePort=0, ipcPort=34195, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004)
2024-11-10T02:11:36,596 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ca18b01a9cd5a63 with lease ID 0xd58ebbc5f537379b: from storage DS-c8b96d83-36b4-4300-bd7e-f5936e3ed9f3 node DatanodeRegistration(127.0.0.1:41963, datanodeUuid=a25e57ba-dbec-409c-92eb-94aa44af277f, infoPort=33623, infoSecurePort=0, ipcPort=34195, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:11:36,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:36,792 WARN  [Thread-1993 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data4/current/BP-292189752-172.17.0.3-1731204694004/current, will proceed with Du for space computation calculation, 
2024-11-10T02:11:36,792 WARN  [Thread-1992 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data3/current/BP-292189752-172.17.0.3-1731204694004/current, will proceed with Du for space computation calculation, 
2024-11-10T02:11:36,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:36,811 WARN  [Thread-1968 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:11:36,812 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4f7592e9a8a91ca with lease ID 0xd58ebbc5f537379c: Processing first storage report for DS-99ba25b5-f1e8-437d-ac51-6f7372aeb25d from datanode DatanodeRegistration(127.0.0.1:36977, datanodeUuid=9589c007-f254-4173-946d-0b774e725934, infoPort=34515, infoSecurePort=0, ipcPort=35395, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004)
2024-11-10T02:11:36,812 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4f7592e9a8a91ca with lease ID 0xd58ebbc5f537379c: from storage DS-99ba25b5-f1e8-437d-ac51-6f7372aeb25d node DatanodeRegistration(127.0.0.1:36977, datanodeUuid=9589c007-f254-4173-946d-0b774e725934, infoPort=34515, infoSecurePort=0, ipcPort=35395, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:11:36,813 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4f7592e9a8a91ca with lease ID 0xd58ebbc5f537379c: Processing first storage report for DS-e0f30e7a-63c1-4b13-b8b1-965cfb722d28 from datanode DatanodeRegistration(127.0.0.1:36977, datanodeUuid=9589c007-f254-4173-946d-0b774e725934, infoPort=34515, infoSecurePort=0, ipcPort=35395, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004)
2024-11-10T02:11:36,813 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4f7592e9a8a91ca with lease ID 0xd58ebbc5f537379c: from storage DS-e0f30e7a-63c1-4b13-b8b1-965cfb722d28 node DatanodeRegistration(127.0.0.1:36977, datanodeUuid=9589c007-f254-4173-946d-0b774e725934, infoPort=34515, infoSecurePort=0, ipcPort=35395, storageInfo=lv=-57;cid=testClusterID;nsid=492127762;c=1731204694004), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:11:36,824 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b
2024-11-10T02:11:36,828 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/zookeeper_0, clientPort=58960, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:11:36,829 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58960
2024-11-10T02:11:36,830 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:36,832 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:36,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:11:36,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:11:36,842 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e with version=8
2024-11-10T02:11:36,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:11:36,844 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:11:36,845 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37519
2024-11-10T02:11:36,846 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37519 connecting to ZooKeeper ensemble=127.0.0.1:58960
2024-11-10T02:11:37,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375190x0, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:11:37,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37519-0x101228346a70000 connected
2024-11-10T02:11:37,114 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:37,116 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:37,118 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:11:37,118 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e, hbase.cluster.distributed=false
2024-11-10T02:11:37,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:11:37,120 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37519
2024-11-10T02:11:37,120 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37519
2024-11-10T02:11:37,120 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37519
2024-11-10T02:11:37,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37519
2024-11-10T02:11:37,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37519
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:11:37,135 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:11:37,136 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36531
2024-11-10T02:11:37,137 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36531 connecting to ZooKeeper ensemble=127.0.0.1:58960
2024-11-10T02:11:37,137 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:37,138 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:37,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365310x0, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:11:37,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365310x0, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:11:37,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36531-0x101228346a70001 connected
2024-11-10T02:11:37,147 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:11:37,148 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:11:37,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:11:37,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:11:37,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36531
2024-11-10T02:11:37,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36531
2024-11-10T02:11:37,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36531
2024-11-10T02:11:37,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36531
2024-11-10T02:11:37,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36531
2024-11-10T02:11:37,170 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:37519
2024-11-10T02:11:37,171 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:11:37,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:11:37,178 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:11:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,188 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:11:37,189 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,37519,1731204696843 from backup master directory
2024-11-10T02:11:37,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:11:37,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:11:37,198 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:11:37,199 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,202 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/hbase.id] with ID: c4952059-1578-48ef-a52d-f0aeb06f093a
2024-11-10T02:11:37,202 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/.tmp/hbase.id
2024-11-10T02:11:37,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:11:37,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:11:37,208 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/.tmp/hbase.id]:[hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/hbase.id]
2024-11-10T02:11:37,218 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:37,218 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:11:37,219 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-10T02:11:37,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:11:37,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:11:37,237 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:11:37,238 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:11:37,238 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:11:37,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:11:37,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:11:37,245 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store
2024-11-10T02:11:37,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:11:37,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:11:37,255 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:37,255 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:11:37,255 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204697255Disabling compacts and flushes for region at 1731204697255Disabling writes for close at 1731204697255Writing region close event to WAL at 1731204697255Closed at 1731204697255
2024-11-10T02:11:37,256 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/.initializing
2024-11-10T02:11:37,256 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/WALs/0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,259 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C37519%2C1731204696843, suffix=, logDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/WALs/0c4d4d227d60,37519,1731204696843, archiveDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/oldWALs, maxLogs=10
2024-11-10T02:11:37,259 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C37519%2C1731204696843.1731204697259
2024-11-10T02:11:37,264 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/WALs/0c4d4d227d60,37519,1731204696843/0c4d4d227d60%2C37519%2C1731204696843.1731204697259
2024-11-10T02:11:37,264 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34515:34515),(127.0.0.1/127.0.0.1:33623:33623)]
2024-11-10T02:11:37,265 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:11:37,265 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:37,265 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,265 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,266 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:11:37,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:11:37,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,268 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:37,268 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:11:37,269 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:11:37,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,269 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:37,269 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:11:37,270 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:11:37,270 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,271 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:37,271 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:11:37,272 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:11:37,272 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,272 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:37,272 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,273 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,273 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,274 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,274 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,275 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:11:37,275 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:11:37,277 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:11:37,278 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796832, jitterRate=0.01322433352470398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:11:37,278 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204697265Initializing all the Stores at 1731204697266 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204697266Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204697266Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204697266Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204697266Cleaning up temporary data from old regions at 1731204697274 (+8 ms)Region opened successfully at 1731204697278 (+4 ms)
2024-11-10T02:11:37,279 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:11:37,281 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e558cc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:11:37,282 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:11:37,282 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:11:37,282 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:11:37,283 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:11:37,283 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:11:37,283 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:11:37,283 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:11:37,286 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:11:37,287 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:11:37,293 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:11:37,293 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:11:37,294 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:11:37,304 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:11:37,304 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:11:37,305 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:11:37,314 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:11:37,315 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:11:37,325 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:11:37,327 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:11:37,335 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:11:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:11:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:11:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,347 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,37519,1731204696843, sessionid=0x101228346a70000, setting cluster-up flag (Was=false)
2024-11-10T02:11:37,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,398 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:11:37,400 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,493 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:11:37,494 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:37,495 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:11:37,497 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:11:37,497 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:11:37,497 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:11:37,497 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,37519,1731204696843
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:11:37,498 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:11:37,498 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:11:37,498 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:11:37,499 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:11:37,499 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:11:37,499 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,499 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:11:37,499 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204727499
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:11:37,500 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:11:37,500 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:11:37,500 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:11:37,501 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,501 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:11:37,501 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:11:37,501 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:11:37,501 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,501 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:11:37,502 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204697502,5,FailOnTimeoutGroup]
2024-11-10T02:11:37,502 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204697502,5,FailOnTimeoutGroup]
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,502 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:11:37,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:11:37,508 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:11:37,508 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e
2024-11-10T02:11:37,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:11:37,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:11:37,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:37,516 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:11:37,517 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:11:37,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,518 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:37,518 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:11:37,519 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:11:37,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,520 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:37,520 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:11:37,521 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:11:37,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,521 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:37,522 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:11:37,523 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:11:37,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:37,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:37,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:11:37,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740
2024-11-10T02:11:37,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740
2024-11-10T02:11:37,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:11:37,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:11:37,526 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:11:37,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:11:37,529 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:11:37,529 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860242, jitterRate=0.09385471045970917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:11:37,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204697515Initializing all the Stores at 1731204697515Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204697516 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204697516Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204697516Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204697516Cleaning up temporary data from old regions at 1731204697526 (+10 ms)Region opened successfully at 1731204697530 (+4 ms)
2024-11-10T02:11:37,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:11:37,530 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:11:37,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:11:37,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:11:37,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:11:37,531 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:11:37,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204697530Disabling compacts and flushes for region at 1731204697530Disabling writes for close at 1731204697530Writing region close event to WAL at 1731204697531 (+1 ms)Closed at 1731204697531
2024-11-10T02:11:37,532 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:11:37,532 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:11:37,532 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:11:37,534 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:11:37,535 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:11:37,562 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(746): ClusterId : c4952059-1578-48ef-a52d-f0aeb06f093a
2024-11-10T02:11:37,562 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:11:37,576 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:11:37,576 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:11:37,589 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:11:37,589 DEBUG [RS:0;0c4d4d227d60:36531 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e08fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:11:37,600 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:36531
2024-11-10T02:11:37,600 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:11:37,600 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:11:37,600 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:11:37,600 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,37519,1731204696843 with port=36531, startcode=1731204697134
2024-11-10T02:11:37,601 DEBUG [RS:0;0c4d4d227d60:36531 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:11:37,602 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50347, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:11:37,603 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,603 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,604 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e
2024-11-10T02:11:37,604 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44877
2024-11-10T02:11:37,604 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:11:37,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:11:37,617 DEBUG [RS:0;0c4d4d227d60:36531 {}] zookeeper.ZKUtil(111): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,617 WARN  [RS:0;0c4d4d227d60:36531 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:11:37,618 INFO  [RS:0;0c4d4d227d60:36531 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:11:37,618 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,618 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,36531,1731204697134]
2024-11-10T02:11:37,621 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:11:37,622 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:11:37,623 INFO  [RS:0;0c4d4d227d60:36531 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:11:37,623 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,623 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:11:37,624 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:11:37,624 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:11:37,624 DEBUG [RS:0;0c4d4d227d60:36531 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,626 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,36531,1731204697134-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:11:37,642 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:11:37,642 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,36531,1731204697134-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,642 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,642 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.Replication(171): 0c4d4d227d60,36531,1731204697134 started
2024-11-10T02:11:37,656 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:37,656 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,36531,1731204697134, RpcServer on 0c4d4d227d60/172.17.0.3:36531, sessionid=0x101228346a70001
2024-11-10T02:11:37,656 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:11:37,656 DEBUG [RS:0;0c4d4d227d60:36531 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,656 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,36531,1731204697134'
2024-11-10T02:11:37,656 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,36531,1731204697134'
2024-11-10T02:11:37,657 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:11:37,658 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:11:37,658 DEBUG [RS:0;0c4d4d227d60:36531 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:11:37,658 INFO  [RS:0;0c4d4d227d60:36531 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:11:37,658 INFO  [RS:0;0c4d4d227d60:36531 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:11:37,685 WARN  [0c4d4d227d60:37519 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:11:37,760 INFO  [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C36531%2C1731204697134, suffix=, logDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134, archiveDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs, maxLogs=32
2024-11-10T02:11:37,761 INFO  [RS:0;0c4d4d227d60:36531 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C36531%2C1731204697134.1731204697761
2024-11-10T02:11:37,769 INFO  [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204697761
2024-11-10T02:11:37,772 DEBUG [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34515:34515),(127.0.0.1/127.0.0.1:33623:33623)]
2024-11-10T02:11:37,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:37,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:37,935 DEBUG [0c4d4d227d60:37519 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:11:37,936 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:37,937 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,36531,1731204697134, state=OPENING
2024-11-10T02:11:37,946 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:11:37,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:11:37,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:11:37,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:11:37,957 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:11:37,957 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,36531,1731204697134}]
2024-11-10T02:11:38,110 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:11:38,112 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36145, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:11:38,115 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:11:38,115 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:11:38,117 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C36531%2C1731204697134.meta, suffix=.meta, logDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134, archiveDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs, maxLogs=32
2024-11-10T02:11:38,117 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C36531%2C1731204697134.meta.1731204698117.meta
2024-11-10T02:11:38,122 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.meta.1731204698117.meta
2024-11-10T02:11:38,123 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34515:34515),(127.0.0.1/127.0.0.1:33623:33623)]
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:11:38,124 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:11:38,124 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:11:38,125 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:11:38,126 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:11:38,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,126 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:38,127 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:11:38,127 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:11:38,127 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,128 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:38,128 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:11:38,128 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:11:38,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,129 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:38,129 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:11:38,129 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:11:38,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,130 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:11:38,130 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:11:38,130 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740
2024-11-10T02:11:38,131 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740
2024-11-10T02:11:38,133 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:11:38,133 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:11:38,133 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:11:38,134 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:11:38,135 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743986, jitterRate=-0.05397412180900574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:11:38,135 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:11:38,136 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204698124Writing region info on filesystem at 1731204698124Initializing all the Stores at 1731204698125 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204698125Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204698125Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204698125Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204698125Cleaning up temporary data from old regions at 1731204698133 (+8 ms)Running coprocessor post-open hooks at 1731204698135 (+2 ms)Region opened successfully at 1731204698136 (+1 ms)
2024-11-10T02:11:38,137 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204698110
2024-11-10T02:11:38,139 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:11:38,139 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:11:38,140 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:38,141 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,36531,1731204697134, state=OPEN
2024-11-10T02:11:38,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:11:38,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:11:38,180 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:38,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:11:38,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:11:38,183 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:11:38,183 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,36531,1731204697134 in 223 msec
2024-11-10T02:11:38,186 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:11:38,186 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 651 msec
2024-11-10T02:11:38,186 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:11:38,186 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:11:38,188 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:11:38,188 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,36531,1731204697134, seqNum=-1]
2024-11-10T02:11:38,188 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:11:38,189 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48787, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:11:38,195 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 698 msec
2024-11-10T02:11:38,195 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204698195, completionTime=-1
2024-11-10T02:11:38,195 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:11:38,195 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:11:38,197 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:11:38,197 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204758197
2024-11-10T02:11:38,197 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204818197
2024-11-10T02:11:38,197 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-10T02:11:38,197 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,198 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,198 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,198 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:37519, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,198 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,198 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,200 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:11:38,201 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.002sec
2024-11-10T02:11:38,201 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:11:38,201 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:11:38,201 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:11:38,201 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:11:38,202 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:11:38,202 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:11:38,202 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:11:38,204 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:11:38,204 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:11:38,204 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,37519,1731204696843-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:11:38,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56251406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:11:38,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,37519,-1 for getting cluster id
2024-11-10T02:11:38,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:11:38,265 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c4952059-1578-48ef-a52d-f0aeb06f093a'
2024-11-10T02:11:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:11:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c4952059-1578-48ef-a52d-f0aeb06f093a"

2024-11-10T02:11:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fefeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:11:38,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,37519,-1]
2024-11-10T02:11:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:11:38,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:11:38,267 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:11:38,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5b2092, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:11:38,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:11:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,36531,1731204697134, seqNum=-1]
2024-11-10T02:11:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:11:38,271 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:11:38,273 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:38,274 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:11:38,276 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:11:38,276 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-10T02:11:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0c4d4d227d60,37519,1731204696843
2024-11-10T02:11:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47b19dcb
2024-11-10T02:11:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-10T02:11:38,279 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-10T02:11:38,279 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-10T02:11:38,279 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-10T02:11:38,280 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:11:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-10T02:11:38,283 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-10T02:11:38,283 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,283 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-10T02:11:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:11:38,284 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-10T02:11:38,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741835_1011 (size=381)
2024-11-10T02:11:38,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741835_1011 (size=381)
2024-11-10T02:11:38,295 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 623ea0c0c13a9c5823b3e3663c4f8c12, NAME => 'TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e
2024-11-10T02:11:38,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741836_1012 (size=64)
2024-11-10T02:11:38,302 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741836_1012 (size=64)
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 623ea0c0c13a9c5823b3e3663c4f8c12, disabling compactions & flushes
2024-11-10T02:11:38,303 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. after waiting 0 ms
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,303 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,303 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 623ea0c0c13a9c5823b3e3663c4f8c12:
Waiting for close lock at 1731204698303Disabling compacts and flushes for region at 1731204698303Disabling writes for close at 1731204698303Writing region close event to WAL at 1731204698303Closed at 1731204698303
2024-11-10T02:11:38,304 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-10T02:11:38,304 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731204698304"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204698304"}]},"ts":"1731204698304"}
2024-11-10T02:11:38,307 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-10T02:11:38,308 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-10T02:11:38,308 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204698308"}]},"ts":"1731204698308"}
2024-11-10T02:11:38,310 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-10T02:11:38,310 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, ASSIGN}]
2024-11-10T02:11:38,311 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, ASSIGN
2024-11-10T02:11:38,312 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, ASSIGN; state=OFFLINE, location=0c4d4d227d60,36531,1731204697134; forceNewPlan=false, retain=false
2024-11-10T02:11:38,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,453 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,453 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,453 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,454 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,463 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=623ea0c0c13a9c5823b3e3663c4f8c12, regionState=OPENING, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:38,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, ASSIGN because future has completed
2024-11-10T02:11:38,466 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134}]
2024-11-10T02:11:38,479 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,479 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,479 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,479 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,480 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,483 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,484 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,484 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,486 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,621 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,621 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 623ea0c0c13a9c5823b3e3663c4f8c12, NAME => 'TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:11:38,621 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,621 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:38,622 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,622 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,623 INFO  [StoreOpener-623ea0c0c13a9c5823b3e3663c4f8c12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 623ea0c0c13a9c5823b3e3663c4f8c12 
2024-11-10T02:11:38,624 INFO  [StoreOpener-623ea0c0c13a9c5823b3e3663c4f8c12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 623ea0c0c13a9c5823b3e3663c4f8c12 columnFamilyName info
2024-11-10T02:11:38,624 DEBUG [StoreOpener-623ea0c0c13a9c5823b3e3663c4f8c12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:38,625 INFO  [StoreOpener-623ea0c0c13a9c5823b3e3663c4f8c12-1 {}] regionserver.HStore(327): Store=623ea0c0c13a9c5823b3e3663c4f8c12/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:38,625 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,625 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,626 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,626 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,626 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,628 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,630 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:11:38,630 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 623ea0c0c13a9c5823b3e3663c4f8c12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848952, jitterRate=0.07949881255626678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:11:38,631 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:38,631 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 623ea0c0c13a9c5823b3e3663c4f8c12:
Running coprocessor pre-open hook at 1731204698622Writing region info on filesystem at 1731204698622Initializing all the Stores at 1731204698622Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204698623 (+1 ms)Cleaning up temporary data from old regions at 1731204698626 (+3 ms)Running coprocessor post-open hooks at 1731204698631 (+5 ms)Region opened successfully at 1731204698631
2024-11-10T02:11:38,632 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., pid=6, masterSystemTime=1731204698618
2024-11-10T02:11:38,634 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,634 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:38,635 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=623ea0c0c13a9c5823b3e3663c4f8c12, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:38,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134 because future has completed
2024-11-10T02:11:38,640 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-10T02:11:38,640 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134 in 172 msec
2024-11-10T02:11:38,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-10T02:11:38,642 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, ASSIGN in 330 msec
2024-11-10T02:11:38,643 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-10T02:11:38,643 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731204698643"}]},"ts":"1731204698643"}
2024-11-10T02:11:38,645 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-10T02:11:38,646 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-10T02:11:38,648 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 366 msec
2024-11-10T02:11:38,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:38,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:38,991 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:11:38,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,993 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,993 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,997 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:38,997 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,036 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-10T02:11:39,422 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-10T02:11:39,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-10T02:11:39,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:39,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:40,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:40,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:41,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:41,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:42,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:42,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:43,621 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-10T02:11:43,622 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-10T02:11:43,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:43,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:44,124 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:11:44,125 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,125 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,126 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,126 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,126 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,126 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,126 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,127 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,146 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,150 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,151 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,151 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,154 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:44,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:44,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:45,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:45,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:46,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:46,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:47,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:47,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37519 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-10T02:11:48,341 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-10T02:11:48,341 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-10T02:11:48,344 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-10T02:11:48,344 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., hostname=0c4d4d227d60,36531,1731204697134, seqNum=2]
2024-11-10T02:11:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:48,359 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:11:48,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/6a5e2f97e6524f09b01afeebf5a543f5 is 1080, key is row0001/info:/1731204708347/Put/seqid=0
2024-11-10T02:11:48,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741837_1013 (size=12509)
2024-11-10T02:11:48,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741837_1013 (size=12509)
2024-11-10T02:11:48,386 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/6a5e2f97e6524f09b01afeebf5a543f5
2024-11-10T02:11:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/6a5e2f97e6524f09b01afeebf5a543f5 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5
2024-11-10T02:11:48,400 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5, entries=7, sequenceid=11, filesize=12.2 K
2024-11-10T02:11:48,401 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 42ms, sequenceid=11, compaction requested=false
2024-11-10T02:11:48,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:48,403 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB
2024-11-10T02:11:48,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/7cbdca0ac62544118e286b7d46b78cb6 is 1080, key is row0008/info:/1731204708361/Put/seqid=0
2024-11-10T02:11:48,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741838_1014 (size=26530)
2024-11-10T02:11:48,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741838_1014 (size=26530)
2024-11-10T02:11:48,414 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/7cbdca0ac62544118e286b7d46b78cb6
2024-11-10T02:11:48,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/7cbdca0ac62544118e286b7d46b78cb6 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6
2024-11-10T02:11:48,425 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6, entries=20, sequenceid=34, filesize=25.9 K
2024-11-10T02:11:48,426 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 23ms, sequenceid=34, compaction requested=false
2024-11-10T02:11:48,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:48,426 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K
2024-11-10T02:11:48,426 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:48,426 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6 because midkey is the same as first or last row
2024-11-10T02:11:48,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:48,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:49,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:49,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:50,420 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:11:50,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/12f68fb66ea34362af8813e4ec9edc41 is 1080, key is row0028/info:/1731204708405/Put/seqid=0
2024-11-10T02:11:50,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741839_1015 (size=12509)
2024-11-10T02:11:50,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741839_1015 (size=12509)
2024-11-10T02:11:50,430 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/12f68fb66ea34362af8813e4ec9edc41
2024-11-10T02:11:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/12f68fb66ea34362af8813e4ec9edc41 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41
2024-11-10T02:11:50,443 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41, entries=7, sequenceid=44, filesize=12.2 K
2024-11-10T02:11:50,443 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8608 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 23ms, sequenceid=44, compaction requested=true
2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6 because midkey is the same as first or last row
2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 623ea0c0c13a9c5823b3e3663c4f8c12:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:11:50,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:50,444 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:11:50,445 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:11:50,445 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 623ea0c0c13a9c5823b3e3663c4f8c12/info is initiating minor compaction (all files)
2024-11-10T02:11:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:50,445 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 623ea0c0c13a9c5823b3e3663c4f8c12/info in TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:50,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-10T02:11:50,446 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp, totalSize=50.3 K
2024-11-10T02:11:50,446 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5e2f97e6524f09b01afeebf5a543f5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731204708347
2024-11-10T02:11:50,448 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cbdca0ac62544118e286b7d46b78cb6, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731204708361
2024-11-10T02:11:50,449 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f68fb66ea34362af8813e4ec9edc41, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731204708405
2024-11-10T02:11:50,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/b53315d555864f6aa4f3598c8210d49f is 1080, key is row0035/info:/1731204710421/Put/seqid=0
2024-11-10T02:11:50,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741840_1016 (size=14663)
2024-11-10T02:11:50,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741840_1016 (size=14663)
2024-11-10T02:11:50,464 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/b53315d555864f6aa4f3598c8210d49f
2024-11-10T02:11:50,468 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 623ea0c0c13a9c5823b3e3663c4f8c12#info#compaction#61 average throughput is 17.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:50,468 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/7d86a30183a04e20a0f2b2e3b9937abb is 1080, key is row0001/info:/1731204708347/Put/seqid=0
2024-11-10T02:11:50,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/b53315d555864f6aa4f3598c8210d49f as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f
2024-11-10T02:11:50,477 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f, entries=9, sequenceid=56, filesize=14.3 K
2024-11-10T02:11:50,478 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=8.41 KB/8608 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 33ms, sequenceid=56, compaction requested=false
2024-11-10T02:11:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.7 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6 because midkey is the same as first or last row
2024-11-10T02:11:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:50,480 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-10T02:11:50,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/957936671cab4652bc96589238a3e501 is 1080, key is row0044/info:/1731204710447/Put/seqid=0
2024-11-10T02:11:50,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741841_1017 (size=41747)
2024-11-10T02:11:50,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741841_1017 (size=41747)
2024-11-10T02:11:50,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741842_1018 (size=15740)
2024-11-10T02:11:50,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741842_1018 (size=15740)
2024-11-10T02:11:50,497 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/957936671cab4652bc96589238a3e501
2024-11-10T02:11:50,499 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/7d86a30183a04e20a0f2b2e3b9937abb as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb
2024-11-10T02:11:50,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/957936671cab4652bc96589238a3e501 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501
2024-11-10T02:11:50,508 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 623ea0c0c13a9c5823b3e3663c4f8c12/info of 623ea0c0c13a9c5823b3e3663c4f8c12 into 7d86a30183a04e20a0f2b2e3b9937abb(size=40.8 K), total size for store is 55.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:50,508 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., storeName=623ea0c0c13a9c5823b3e3663c4f8c12/info, priority=13, startTime=1731204710444; duration=0sec
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=55.1 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb because midkey is the same as first or last row
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=55.1 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,508 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb because midkey is the same as first or last row
2024-11-10T02:11:50,509 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=55.1 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,509 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,509 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb because midkey is the same as first or last row
2024-11-10T02:11:50,509 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:50,509 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 623ea0c0c13a9c5823b3e3663c4f8c12:info
2024-11-10T02:11:50,511 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501, entries=10, sequenceid=69, filesize=15.4 K
2024-11-10T02:11:50,512 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=11.56 KB/11836 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 32ms, sequenceid=69, compaction requested=true
2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=70.5 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb because midkey is the same as first or last row
2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 623ea0c0c13a9c5823b3e3663c4f8c12:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:11:50,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:50,512 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:11:50,514 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 72150 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:11:50,514 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 623ea0c0c13a9c5823b3e3663c4f8c12/info is initiating minor compaction (all files)
2024-11-10T02:11:50,514 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 623ea0c0c13a9c5823b3e3663c4f8c12/info in TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:50,514 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp, totalSize=70.5 K
2024-11-10T02:11:50,515 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d86a30183a04e20a0f2b2e3b9937abb, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1731204708347
2024-11-10T02:11:50,515 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting b53315d555864f6aa4f3598c8210d49f, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731204710421
2024-11-10T02:11:50,516 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 957936671cab4652bc96589238a3e501, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731204710447
2024-11-10T02:11:50,530 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 623ea0c0c13a9c5823b3e3663c4f8c12#info#compaction#63 average throughput is 18.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:50,531 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/1f7ed462e2be4e93ae340735850441da is 1080, key is row0001/info:/1731204708347/Put/seqid=0
2024-11-10T02:11:50,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741843_1019 (size=62381)
2024-11-10T02:11:50,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741843_1019 (size=62381)
2024-11-10T02:11:50,547 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/1f7ed462e2be4e93ae340735850441da as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da
2024-11-10T02:11:50,556 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 623ea0c0c13a9c5823b3e3663c4f8c12/info of 623ea0c0c13a9c5823b3e3663c4f8c12 into 1f7ed462e2be4e93ae340735850441da(size=60.9 K), total size for store is 60.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:50,556 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., storeName=623ea0c0c13a9c5823b3e3663c4f8c12/info, priority=13, startTime=1731204710512; duration=0sec
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.9 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.9 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=60.9 K, sizeToCheck=16.0 K
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:50,556 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 623ea0c0c13a9c5823b3e3663c4f8c12:info
2024-11-10T02:11:50,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:50,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:51,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:51,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,512 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-10T02:11:52,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/17b6428218bd48c78202f71753a90e96 is 1080, key is row0054/info:/1731204710481/Put/seqid=0
2024-11-10T02:11:52,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741844_1020 (size=17894)
2024-11-10T02:11:52,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741844_1020 (size=17894)
2024-11-10T02:11:52,528 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/17b6428218bd48c78202f71753a90e96
2024-11-10T02:11:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/17b6428218bd48c78202f71753a90e96 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96
2024-11-10T02:11:52,546 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96, entries=12, sequenceid=86, filesize=17.5 K
2024-11-10T02:11:52,548 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 36ms, sequenceid=86, compaction requested=false
2024-11-10T02:11:52,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,549 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:52,549 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-10T02:11:52,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/688dc85ab05e41d184b9f8a62a84b4db is 1080, key is row0066/info:/1731204712513/Put/seqid=0
2024-11-10T02:11:52,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741845_1021 (size=15740)
2024-11-10T02:11:52,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741845_1021 (size=15740)
2024-11-10T02:11:52,570 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/688dc85ab05e41d184b9f8a62a84b4db
2024-11-10T02:11:52,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/688dc85ab05e41d184b9f8a62a84b4db as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db
2024-11-10T02:11:52,583 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db, entries=10, sequenceid=99, filesize=15.4 K
2024-11-10T02:11:52,584 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=12.61 KB/12912 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 35ms, sequenceid=99, compaction requested=true
2024-11-10T02:11:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.8 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,584 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:52,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 623ea0c0c13a9c5823b3e3663c4f8c12:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:11:52,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:52,585 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:11:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,585 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-10T02:11:52,586 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 96015 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:11:52,586 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 623ea0c0c13a9c5823b3e3663c4f8c12/info is initiating minor compaction (all files)
2024-11-10T02:11:52,586 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 623ea0c0c13a9c5823b3e3663c4f8c12/info in TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:52,586 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp, totalSize=93.8 K
2024-11-10T02:11:52,587 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7ed462e2be4e93ae340735850441da, keycount=53, bloomtype=ROW, size=60.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731204708347
2024-11-10T02:11:52,587 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b6428218bd48c78202f71753a90e96, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731204710481
2024-11-10T02:11:52,588 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 688dc85ab05e41d184b9f8a62a84b4db, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731204712513
2024-11-10T02:11:52,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/2e3d91bd147543a9a6197e0e9c61d80a is 1080, key is row0076/info:/1731204712551/Put/seqid=0
2024-11-10T02:11:52,608 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 623ea0c0c13a9c5823b3e3663c4f8c12#info#compaction#67 average throughput is 25.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:52,609 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/56b4ce752e974b12a2145cf798958f1b is 1080, key is row0001/info:/1731204708347/Put/seqid=0
2024-11-10T02:11:52,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741846_1022 (size=18987)
2024-11-10T02:11:52,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741846_1022 (size=18987)
2024-11-10T02:11:52,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741847_1023 (size=86250)
2024-11-10T02:11:52,626 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741847_1023 (size=86250)
2024-11-10T02:11:52,628 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/2e3d91bd147543a9a6197e0e9c61d80a
2024-11-10T02:11:52,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/2e3d91bd147543a9a6197e0e9c61d80a as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/2e3d91bd147543a9a6197e0e9c61d80a
2024-11-10T02:11:52,635 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/56b4ce752e974b12a2145cf798958f1b as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b
2024-11-10T02:11:52,641 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/2e3d91bd147543a9a6197e0e9c61d80a, entries=13, sequenceid=115, filesize=18.5 K
2024-11-10T02:11:52,642 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=8.41 KB/8608 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 57ms, sequenceid=115, compaction requested=false
2024-11-10T02:11:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.3 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da because midkey is the same as first or last row
2024-11-10T02:11:52,643 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 623ea0c0c13a9c5823b3e3663c4f8c12/info of 623ea0c0c13a9c5823b3e3663c4f8c12 into 56b4ce752e974b12a2145cf798958f1b(size=84.2 K), total size for store is 102.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:52,643 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 623ea0c0c13a9c5823b3e3663c4f8c12:

2024-11-10T02:11:52,643 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., storeName=623ea0c0c13a9c5823b3e3663c4f8c12/info, priority=13, startTime=1731204712584; duration=0sec
2024-11-10T02:11:52,643 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,643 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,644 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,644 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,644 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.8 K, sizeToCheck=16.0 K
2024-11-10T02:11:52,644 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-10T02:11:52,645 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:52,645 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:52,645 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 623ea0c0c13a9c5823b3e3663c4f8c12:info
2024-11-10T02:11:52,646 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] assignment.AssignmentManager(1355): Split request from 0c4d4d227d60,36531,1731204697134, parent={ENCODED => 623ea0c0c13a9c5823b3e3663c4f8c12, NAME => 'TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-10T02:11:52,651 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:52,655 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=623ea0c0c13a9c5823b3e3663c4f8c12, daughterA=f6e34468bc8d4c76535cd1cf0979e04f, daughterB=05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:52,656 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=623ea0c0c13a9c5823b3e3663c4f8c12, daughterA=f6e34468bc8d4c76535cd1cf0979e04f, daughterB=05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:52,656 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=623ea0c0c13a9c5823b3e3663c4f8c12, daughterA=f6e34468bc8d4c76535cd1cf0979e04f, daughterB=05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:52,656 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=623ea0c0c13a9c5823b3e3663c4f8c12, daughterA=f6e34468bc8d4c76535cd1cf0979e04f, daughterB=05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:52,665 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, UNASSIGN}]
2024-11-10T02:11:52,666 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, UNASSIGN
2024-11-10T02:11:52,668 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=623ea0c0c13a9c5823b3e3663c4f8c12, regionState=CLOSING, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:52,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, UNASSIGN because future has completed
2024-11-10T02:11:52,671 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-10T02:11:52,671 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134}]
2024-11-10T02:11:52,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:52,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:52,829 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,829 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-10T02:11:52,829 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 623ea0c0c13a9c5823b3e3663c4f8c12, disabling compactions & flushes
2024-11-10T02:11:52,830 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:52,830 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:52,830 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. after waiting 0 ms
2024-11-10T02:11:52,830 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:52,830 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 623ea0c0c13a9c5823b3e3663c4f8c12 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-10T02:11:52,834 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/582bb0c469644ff6a4338b03a0a6e5ae is 1080, key is row0089/info:/1731204712587/Put/seqid=0
2024-11-10T02:11:52,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741848_1024 (size=13586)
2024-11-10T02:11:52,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741848_1024 (size=13586)
2024-11-10T02:11:52,839 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/582bb0c469644ff6a4338b03a0a6e5ae
2024-11-10T02:11:52,844 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/.tmp/info/582bb0c469644ff6a4338b03a0a6e5ae as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/582bb0c469644ff6a4338b03a0a6e5ae
2024-11-10T02:11:52,849 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/582bb0c469644ff6a4338b03a0a6e5ae, entries=8, sequenceid=127, filesize=13.3 K
2024-11-10T02:11:52,850 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 20ms, sequenceid=127, compaction requested=true
2024-11-10T02:11:52,851 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db] to archive
2024-11-10T02:11:52,852 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:11:52,854 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/6a5e2f97e6524f09b01afeebf5a543f5
2024-11-10T02:11:52,855 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7cbdca0ac62544118e286b7d46b78cb6
2024-11-10T02:11:52,856 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/7d86a30183a04e20a0f2b2e3b9937abb
2024-11-10T02:11:52,857 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/12f68fb66ea34362af8813e4ec9edc41
2024-11-10T02:11:52,858 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/b53315d555864f6aa4f3598c8210d49f
2024-11-10T02:11:52,859 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/1f7ed462e2be4e93ae340735850441da
2024-11-10T02:11:52,860 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/957936671cab4652bc96589238a3e501
2024-11-10T02:11:52,861 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/17b6428218bd48c78202f71753a90e96
2024-11-10T02:11:52,862 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/688dc85ab05e41d184b9f8a62a84b4db
2024-11-10T02:11:52,869 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-10T02:11:52,870 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.
2024-11-10T02:11:52,870 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 623ea0c0c13a9c5823b3e3663c4f8c12:
Waiting for close lock at 1731204712829Running coprocessor pre-close hooks at 1731204712829Disabling compacts and flushes for region at 1731204712829Disabling writes for close at 1731204712830 (+1 ms)Obtaining lock to block concurrent updates at 1731204712830Preparing flush snapshotting stores in 623ea0c0c13a9c5823b3e3663c4f8c12 at 1731204712830Finished memstore snapshotting TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., syncing WAL and waiting on mvcc, flushsize=dataSize=8608, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1731204712830Flushing stores of TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. at 1731204712831 (+1 ms)Flushing 623ea0c0c13a9c5823b3e3663c4f8c12/info: creating writer at 1731204712831Flushing 623ea0c0c13a9c5823b3e3663c4f8c12/info: appending metadata at 1731204712833 (+2 ms)Flushing 623ea0c0c13a9c5823b3e3663c4f8c12/info: closing flushed file at 1731204712833Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@585d54ba: reopening flushed file at 1731204712843 (+10 ms)Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for 623ea0c0c13a9c5823b3e3663c4f8c12 in 20ms, sequenceid=127, compaction requested=true at 1731204712850 (+7 ms)Writing region close event to WAL at 1731204712865 (+15 ms)Running coprocessor post-close hooks at 1731204712870 (+5 ms)Closed at 1731204712870
2024-11-10T02:11:52,873 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,873 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=623ea0c0c13a9c5823b3e3663c4f8c12, regionState=CLOSED
2024-11-10T02:11:52,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134 because future has completed
2024-11-10T02:11:52,879 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-10T02:11:52,879 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 623ea0c0c13a9c5823b3e3663c4f8c12, server=0c4d4d227d60,36531,1731204697134 in 206 msec
2024-11-10T02:11:52,880 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-10T02:11:52,880 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=623ea0c0c13a9c5823b3e3663c4f8c12, UNASSIGN in 215 msec
2024-11-10T02:11:52,888 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:52,891 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=623ea0c0c13a9c5823b3e3663c4f8c12, threads=3
2024-11-10T02:11:52,893 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,893 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/2e3d91bd147543a9a6197e0e9c61d80a for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,893 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/582bb0c469644ff6a4338b03a0a6e5ae for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,903 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/2e3d91bd147543a9a6197e0e9c61d80a, top=true
2024-11-10T02:11:52,903 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/582bb0c469644ff6a4338b03a0a6e5ae, top=true
2024-11-10T02:11:52,914 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a for child: 05915fec36f2ccb3940bfed44b48c54a, parent: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,914 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae for child: 05915fec36f2ccb3940bfed44b48c54a, parent: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741849_1025 (size=27)
2024-11-10T02:11:52,914 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/2e3d91bd147543a9a6197e0e9c61d80a for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,914 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/582bb0c469644ff6a4338b03a0a6e5ae for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741849_1025 (size=27)
2024-11-10T02:11:52,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741850_1026 (size=27)
2024-11-10T02:11:52,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741850_1026 (size=27)
2024-11-10T02:11:52,930 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b for region: 623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:11:52,933 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 623ea0c0c13a9c5823b3e3663c4f8c12 Daughter A: [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12] storefiles, Daughter B: [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae] storefiles.
2024-11-10T02:11:52,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741851_1027 (size=71)
2024-11-10T02:11:52,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741851_1027 (size=71)
2024-11-10T02:11:52,943 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:52,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741852_1028 (size=71)
2024-11-10T02:11:52,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741852_1028 (size=71)
2024-11-10T02:11:52,956 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:52,964 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-10T02:11:52,965 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-10T02:11:52,968 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731204712967"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731204712967"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731204712967"}]},"ts":"1731204712967"}
2024-11-10T02:11:52,968 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731204712967"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204712967"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731204712967"}]},"ts":"1731204712967"}
2024-11-10T02:11:52,968 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731204712967"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731204712967"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731204712967"}]},"ts":"1731204712967"}
2024-11-10T02:11:52,984 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f6e34468bc8d4c76535cd1cf0979e04f, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a, ASSIGN}]
2024-11-10T02:11:52,986 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f6e34468bc8d4c76535cd1cf0979e04f, ASSIGN
2024-11-10T02:11:52,986 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a, ASSIGN
2024-11-10T02:11:52,987 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f6e34468bc8d4c76535cd1cf0979e04f, ASSIGN; state=SPLITTING_NEW, location=0c4d4d227d60,36531,1731204697134; forceNewPlan=false, retain=false
2024-11-10T02:11:52,987 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a, ASSIGN; state=SPLITTING_NEW, location=0c4d4d227d60,36531,1731204697134; forceNewPlan=false, retain=false
2024-11-10T02:11:53,137 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f6e34468bc8d4c76535cd1cf0979e04f, regionState=OPENING, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:53,137 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=05915fec36f2ccb3940bfed44b48c54a, regionState=OPENING, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:53,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f6e34468bc8d4c76535cd1cf0979e04f, ASSIGN because future has completed
2024-11-10T02:11:53,140 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e34468bc8d4c76535cd1cf0979e04f, server=0c4d4d227d60,36531,1731204697134}]
2024-11-10T02:11:53,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a, ASSIGN because future has completed
2024-11-10T02:11:53,142 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134}]
2024-11-10T02:11:53,296 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:11:53,296 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f6e34468bc8d4c76535cd1cf0979e04f, NAME => 'TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-10T02:11:53,297 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,297 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:53,297 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,297 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,298 INFO  [StoreOpener-f6e34468bc8d4c76535cd1cf0979e04f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f6e34468bc8d4c76535cd1cf0979e04f 
2024-11-10T02:11:53,299 INFO  [StoreOpener-f6e34468bc8d4c76535cd1cf0979e04f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6e34468bc8d4c76535cd1cf0979e04f columnFamilyName info
2024-11-10T02:11:53,299 DEBUG [StoreOpener-f6e34468bc8d4c76535cd1cf0979e04f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:53,313 DEBUG [StoreOpener-f6e34468bc8d4c76535cd1cf0979e04f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-bottom
2024-11-10T02:11:53,313 INFO  [StoreOpener-f6e34468bc8d4c76535cd1cf0979e04f-1 {}] regionserver.HStore(327): Store=f6e34468bc8d4c76535cd1cf0979e04f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:53,313 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,314 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,315 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,316 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,316 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,317 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,318 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened f6e34468bc8d4c76535cd1cf0979e04f; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=883899, jitterRate=0.1239367127418518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:11:53,318 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:11:53,319 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f6e34468bc8d4c76535cd1cf0979e04f:
Running coprocessor pre-open hook at 1731204713297Writing region info on filesystem at 1731204713297Initializing all the Stores at 1731204713298 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204713298Cleaning up temporary data from old regions at 1731204713316 (+18 ms)Running coprocessor post-open hooks at 1731204713318 (+2 ms)Region opened successfully at 1731204713319 (+1 ms)
2024-11-10T02:11:53,320 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f., pid=12, masterSystemTime=1731204713293
2024-11-10T02:11:53,320 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store f6e34468bc8d4c76535cd1cf0979e04f:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:11:53,320 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:53,320 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-10T02:11:53,321 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:11:53,321 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): f6e34468bc8d4c76535cd1cf0979e04f/info is initiating minor compaction (all files)
2024-11-10T02:11:53,321 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f6e34468bc8d4c76535cd1cf0979e04f/info in TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:11:53,321 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-bottom] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/.tmp, totalSize=84.2 K
2024-11-10T02:11:53,322 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12, keycount=37, bloomtype=ROW, size=84.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731204708347
2024-11-10T02:11:53,322 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:11:53,322 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:11:53,322 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:11:53,322 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 05915fec36f2ccb3940bfed44b48c54a, NAME => 'TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-10T02:11:53,323 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,323 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:11:53,323 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,323 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,323 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=f6e34468bc8d4c76535cd1cf0979e04f, regionState=OPEN, openSeqNum=131, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:53,324 INFO  [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 05915fec36f2ccb3940bfed44b48c54a 
2024-11-10T02:11:53,325 INFO  [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05915fec36f2ccb3940bfed44b48c54a columnFamilyName info
2024-11-10T02:11:53,325 DEBUG [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:11:53,325 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-10T02:11:53,325 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-10T02:11:53,325 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-10T02:11:53,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e34468bc8d4c76535cd1cf0979e04f, server=0c4d4d227d60,36531,1731204697134 because future has completed
2024-11-10T02:11:53,332 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-10T02:11:53,332 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure f6e34468bc8d4c76535cd1cf0979e04f, server=0c4d4d227d60,36531,1731204697134 in 188 msec
2024-11-10T02:11:53,334 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=f6e34468bc8d4c76535cd1cf0979e04f, ASSIGN in 348 msec
2024-11-10T02:11:53,338 DEBUG [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-top
2024-11-10T02:11:53,343 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f6e34468bc8d4c76535cd1cf0979e04f#info#compaction#69 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:53,344 DEBUG [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a
2024-11-10T02:11:53,344 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/.tmp/info/3178798722c74f369807c98a85e15c0d is 1080, key is row0001/info:/1731204708347/Put/seqid=0
2024-11-10T02:11:53,350 DEBUG [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae
2024-11-10T02:11:53,350 INFO  [StoreOpener-05915fec36f2ccb3940bfed44b48c54a-1 {}] regionserver.HStore(327): Store=05915fec36f2ccb3940bfed44b48c54a/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:11:53,350 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,351 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/167c15f1d84747f0966a0259ea24a968 is 193, key is TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a./info:regioninfo/1731204713137/Put/seqid=0
2024-11-10T02:11:53,352 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,353 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,353 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,355 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,356 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 05915fec36f2ccb3940bfed44b48c54a; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735130, jitterRate=-0.06523388624191284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-10T02:11:53,356 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:11:53,356 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 05915fec36f2ccb3940bfed44b48c54a:
Running coprocessor pre-open hook at 1731204713323Writing region info on filesystem at 1731204713323Initializing all the Stores at 1731204713323Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204713323Cleaning up temporary data from old regions at 1731204713353 (+30 ms)Running coprocessor post-open hooks at 1731204713356 (+3 ms)Region opened successfully at 1731204713356
2024-11-10T02:11:53,357 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., pid=13, masterSystemTime=1731204713293
2024-11-10T02:11:53,357 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 2
2024-11-10T02:11:53,357 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:53,357 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:11:53,358 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:11:53,359 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:11:53,359 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:11:53,359 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-top, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=116.0 K
2024-11-10T02:11:53,359 DEBUG [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:11:53,359 INFO  [RS_OPEN_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:11:53,360 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12, keycount=37, bloomtype=ROW, size=84.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731204708347
2024-11-10T02:11:53,360 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731204712551
2024-11-10T02:11:53,360 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=05915fec36f2ccb3940bfed44b48c54a, regionState=OPEN, openSeqNum=131, regionLocation=0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:53,360 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731204712587
2024-11-10T02:11:53,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134 because future has completed
2024-11-10T02:11:53,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741853_1029 (size=70862)
2024-11-10T02:11:53,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741853_1029 (size=70862)
2024-11-10T02:11:53,385 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37519 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=0c4d4d227d60,36531,1731204697134, table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-10T02:11:53,388 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-10T02:11:53,388 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134 in 244 msec
2024-11-10T02:11:53,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-10T02:11:53,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=05915fec36f2ccb3940bfed44b48c54a, ASSIGN in 404 msec
2024-11-10T02:11:53,394 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=623ea0c0c13a9c5823b3e3663c4f8c12, daughterA=f6e34468bc8d4c76535cd1cf0979e04f, daughterB=05915fec36f2ccb3940bfed44b48c54a in 741 msec
2024-11-10T02:11:53,402 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/.tmp/info/3178798722c74f369807c98a85e15c0d as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/3178798722c74f369807c98a85e15c0d
2024-11-10T02:11:53,403 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#71 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:11:53,403 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/cf7e1d742dac4e329566c6f78b92354e is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:11:53,406 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741854_1030 (size=9847)
2024-11-10T02:11:53,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741854_1030 (size=9847)
2024-11-10T02:11:53,409 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/167c15f1d84747f0966a0259ea24a968
2024-11-10T02:11:53,409 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in f6e34468bc8d4c76535cd1cf0979e04f/info of f6e34468bc8d4c76535cd1cf0979e04f into 3178798722c74f369807c98a85e15c0d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:53,409 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f6e34468bc8d4c76535cd1cf0979e04f:

2024-11-10T02:11:53,409 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f., storeName=f6e34468bc8d4c76535cd1cf0979e04f/info, priority=15, startTime=1731204713320; duration=0sec
2024-11-10T02:11:53,409 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:53,409 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f6e34468bc8d4c76535cd1cf0979e04f:info
2024-11-10T02:11:53,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741855_1031 (size=42984)
2024-11-10T02:11:53,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741855_1031 (size=42984)
2024-11-10T02:11:53,433 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/cf7e1d742dac4e329566c6f78b92354e as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/cf7e1d742dac4e329566c6f78b92354e
2024-11-10T02:11:53,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/ns/c866c54cf2fc499fb82c41c8300a896b is 43, key is default/ns:d/1731204698190/Put/seqid=0
2024-11-10T02:11:53,441 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into cf7e1d742dac4e329566c6f78b92354e(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:11:53,441 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:11:53,441 INFO  [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204713357; duration=0sec
2024-11-10T02:11:53,441 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:11:53,441 DEBUG [RS:0;0c4d4d227d60:36531-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:11:53,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741856_1032 (size=5153)
2024-11-10T02:11:53,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741856_1032 (size=5153)
2024-11-10T02:11:53,450 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/ns/c866c54cf2fc499fb82c41c8300a896b
2024-11-10T02:11:53,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/table/9ace81f31e034407b1500839284c2799 is 65, key is TestLogRolling-testLogRolling/table:state/1731204698643/Put/seqid=0
2024-11-10T02:11:53,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741857_1033 (size=5340)
2024-11-10T02:11:53,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741857_1033 (size=5340)
2024-11-10T02:11:53,474 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/table/9ace81f31e034407b1500839284c2799
2024-11-10T02:11:53,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/167c15f1d84747f0966a0259ea24a968 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/info/167c15f1d84747f0966a0259ea24a968
2024-11-10T02:11:53,487 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/info/167c15f1d84747f0966a0259ea24a968, entries=30, sequenceid=17, filesize=9.6 K
2024-11-10T02:11:53,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/ns/c866c54cf2fc499fb82c41c8300a896b as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/ns/c866c54cf2fc499fb82c41c8300a896b
2024-11-10T02:11:53,494 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/ns/c866c54cf2fc499fb82c41c8300a896b, entries=2, sequenceid=17, filesize=5.0 K
2024-11-10T02:11:53,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/table/9ace81f31e034407b1500839284c2799 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/table/9ace81f31e034407b1500839284c2799
2024-11-10T02:11:53,501 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/table/9ace81f31e034407b1500839284c2799, entries=2, sequenceid=17, filesize=5.2 K
2024-11-10T02:11:53,502 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 177ms, sequenceid=17, compaction requested=false
2024-11-10T02:11:53,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-10T02:11:53,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:53,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:50564 deadline: 1731204724617, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. is not online on 0c4d4d227d60,36531,1731204697134
2024-11-10T02:11:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., hostname=0c4d4d227d60,36531,1731204697134, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., hostname=0c4d4d227d60,36531,1731204697134, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. is not online on 0c4d4d227d60,36531,1731204697134
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-10T02:11:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., hostname=0c4d4d227d60,36531,1731204697134, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12. is not online on 0c4d4d227d60,36531,1731204697134
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-10T02:11:54,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731204698279.623ea0c0c13a9c5823b3e3663c4f8c12., hostname=0c4d4d227d60,36531,1731204697134, seqNum=2 from cache
2024-11-10T02:11:54,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:54,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:55,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:55,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:56,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:56,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:57,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:57,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:57,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,871 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,872 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,906 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:57,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,429 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-10T02:11:58,429 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,430 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,431 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,432 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,460 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,461 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,464 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,465 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,465 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-10T02:11:58,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:58,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:59,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:11:59,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:00,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:00,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:01,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:01,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:02,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:02,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:03,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:03,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., hostname=0c4d4d227d60,36531,1731204697134, seqNum=131]
2024-11-10T02:12:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:04,699 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:12:04,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/426c176cb35b4826b1c5d8c9ba22f439 is 1080, key is row0097/info:/1731204724688/Put/seqid=0
2024-11-10T02:12:04,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741858_1034 (size=12516)
2024-11-10T02:12:04,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741858_1034 (size=12516)
2024-11-10T02:12:04,713 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/426c176cb35b4826b1c5d8c9ba22f439
2024-11-10T02:12:04,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/426c176cb35b4826b1c5d8c9ba22f439 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439
2024-11-10T02:12:04,726 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439, entries=7, sequenceid=141, filesize=12.2 K
2024-11-10T02:12:04,727 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 05915fec36f2ccb3940bfed44b48c54a in 28ms, sequenceid=141, compaction requested=false
2024-11-10T02:12:04,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:04,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-10T02:12:04,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/a607995d4359441d8d0ea481a8076615 is 1080, key is row0104/info:/1731204724700/Put/seqid=0
2024-11-10T02:12:04,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741859_1035 (size=19000)
2024-11-10T02:12:04,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741859_1035 (size=19000)
2024-11-10T02:12:04,737 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/a607995d4359441d8d0ea481a8076615
2024-11-10T02:12:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/a607995d4359441d8d0ea481a8076615 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615
2024-11-10T02:12:04,748 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615, entries=13, sequenceid=157, filesize=18.6 K
2024-11-10T02:12:04,749 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=9.46 KB/9684 for 05915fec36f2ccb3940bfed44b48c54a in 22ms, sequenceid=157, compaction requested=true
2024-11-10T02:12:04,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:04,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:04,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:04,749 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:12:04,751 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 74500 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:12:04,751 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:04,751 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:04,751 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-10T02:12:04,751 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/cf7e1d742dac4e329566c6f78b92354e, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=72.8 K
2024-11-10T02:12:04,751 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf7e1d742dac4e329566c6f78b92354e, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731204710503
2024-11-10T02:12:04,752 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 426c176cb35b4826b1c5d8c9ba22f439, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731204724688
2024-11-10T02:12:04,752 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting a607995d4359441d8d0ea481a8076615, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731204724700
2024-11-10T02:12:04,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/e1ff3cb5341340829189ea509ee23bc0 is 1080, key is row0117/info:/1731204724729/Put/seqid=0
2024-11-10T02:12:04,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741860_1036 (size=15750)
2024-11-10T02:12:04,759 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741860_1036 (size=15750)
2024-11-10T02:12:04,760 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/e1ff3cb5341340829189ea509ee23bc0
2024-11-10T02:12:04,764 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#77 average throughput is 28.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:04,765 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/6b169e5350bd4e229db979a7803ef177 is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:04,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/e1ff3cb5341340829189ea509ee23bc0 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0
2024-11-10T02:12:04,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741861_1037 (size=64714)
2024-11-10T02:12:04,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741861_1037 (size=64714)
2024-11-10T02:12:04,773 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0, entries=10, sequenceid=170, filesize=15.4 K
2024-11-10T02:12:04,773 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2152 for 05915fec36f2ccb3940bfed44b48c54a in 22ms, sequenceid=170, compaction requested=false
2024-11-10T02:12:04,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:04,776 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/6b169e5350bd4e229db979a7803ef177 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/6b169e5350bd4e229db979a7803ef177
2024-11-10T02:12:04,782 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into 6b169e5350bd4e229db979a7803ef177(size=63.2 K), total size for store is 78.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:04,782 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:04,782 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204724749; duration=0sec
2024-11-10T02:12:04,782 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:04,782 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:04,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:04,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:05,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:05,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:06,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:06,766 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:12:06,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/fd4442ac104c4fa3976cff7abf4472ce is 1080, key is row0127/info:/1731204724752/Put/seqid=0
2024-11-10T02:12:06,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741862_1038 (size=12516)
2024-11-10T02:12:06,777 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741862_1038 (size=12516)
2024-11-10T02:12:06,778 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/fd4442ac104c4fa3976cff7abf4472ce
2024-11-10T02:12:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/fd4442ac104c4fa3976cff7abf4472ce as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce
2024-11-10T02:12:06,790 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce, entries=7, sequenceid=181, filesize=12.2 K
2024-11-10T02:12:06,791 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 05915fec36f2ccb3940bfed44b48c54a in 24ms, sequenceid=181, compaction requested=true
2024-11-10T02:12:06,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:06,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:06,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:06,791 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:12:06,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:06,792 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-10T02:12:06,792 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 92980 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:12:06,792 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:06,792 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:06,792 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/6b169e5350bd4e229db979a7803ef177, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=90.8 K
2024-11-10T02:12:06,793 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b169e5350bd4e229db979a7803ef177, keycount=55, bloomtype=ROW, size=63.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731204710503
2024-11-10T02:12:06,793 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ff3cb5341340829189ea509ee23bc0, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731204724729
2024-11-10T02:12:06,793 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4442ac104c4fa3976cff7abf4472ce, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731204724752
2024-11-10T02:12:06,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/b2dcc38daf994f31a9a638fd25c3cbfc is 1080, key is row0134/info:/1731204726767/Put/seqid=0
2024-11-10T02:12:06,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:06,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:06,824 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-10T02:12:06,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741863_1039 (size=17906)
2024-11-10T02:12:06,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741863_1039 (size=17906)
2024-11-10T02:12:06,841 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/b2dcc38daf994f31a9a638fd25c3cbfc
2024-11-10T02:12:06,841 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#80 average throughput is 24.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:06,842 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/d8d5688837c54de5ab247fb11be730d2 is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:06,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/b2dcc38daf994f31a9a638fd25c3cbfc as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc
2024-11-10T02:12:06,860 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc, entries=12, sequenceid=196, filesize=17.5 K
2024-11-10T02:12:06,862 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=15.76 KB/16140 for 05915fec36f2ccb3940bfed44b48c54a in 69ms, sequenceid=196, compaction requested=false
2024-11-10T02:12:06,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:06,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741864_1040 (size=83215)
2024-11-10T02:12:06,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741864_1040 (size=83215)
2024-11-10T02:12:06,879 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/d8d5688837c54de5ab247fb11be730d2 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d8d5688837c54de5ab247fb11be730d2
2024-11-10T02:12:06,887 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into d8d5688837c54de5ab247fb11be730d2(size=81.3 K), total size for store is 98.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:06,887 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:06,887 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204726791; duration=0sec
2024-11-10T02:12:06,887 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:06,887 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:07,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:07,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:08,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:08,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:08,860 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-10T02:12:08,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dc3fe3914d5343a8a8aff50a473b5496 is 1080, key is row0146/info:/1731204726793/Put/seqid=0
2024-11-10T02:12:08,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741865_1041 (size=22238)
2024-11-10T02:12:08,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741865_1041 (size=22238)
2024-11-10T02:12:08,876 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dc3fe3914d5343a8a8aff50a473b5496
2024-11-10T02:12:08,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dc3fe3914d5343a8a8aff50a473b5496 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496
2024-11-10T02:12:08,887 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496, entries=16, sequenceid=216, filesize=21.7 K
2024-11-10T02:12:08,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for 05915fec36f2ccb3940bfed44b48c54a in 28ms, sequenceid=216, compaction requested=true
2024-11-10T02:12:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:08,888 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:12:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:08,888 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-10T02:12:08,889 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123359 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:12:08,889 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:08,889 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:08,890 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d8d5688837c54de5ab247fb11be730d2, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=120.5 K
2024-11-10T02:12:08,890 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d5688837c54de5ab247fb11be730d2, keycount=72, bloomtype=ROW, size=81.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731204710503
2024-11-10T02:12:08,890 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2dcc38daf994f31a9a638fd25c3cbfc, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731204726767
2024-11-10T02:12:08,891 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc3fe3914d5343a8a8aff50a473b5496, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731204726793
2024-11-10T02:12:08,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/16f75548e1c44f21985f083ac0076358 is 1080, key is row0162/info:/1731204728862/Put/seqid=0
2024-11-10T02:12:08,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741866_1042 (size=20078)
2024-11-10T02:12:08,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741866_1042 (size=20078)
2024-11-10T02:12:08,899 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/16f75548e1c44f21985f083ac0076358
2024-11-10T02:12:08,905 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#83 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:08,906 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/98511745aba6440ab018613d59dfb734 is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:08,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/16f75548e1c44f21985f083ac0076358 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358
2024-11-10T02:12:08,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358, entries=14, sequenceid=233, filesize=19.6 K
2024-11-10T02:12:08,914 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=10.51 KB/10760 for 05915fec36f2ccb3940bfed44b48c54a in 26ms, sequenceid=233, compaction requested=false
2024-11-10T02:12:08,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:08,916 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-10T02:12:08,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dfd73d3cab49479db26091ea1d448da3 is 1080, key is row0176/info:/1731204728890/Put/seqid=0
2024-11-10T02:12:08,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741868_1044 (size=17906)
2024-11-10T02:12:08,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741867_1043 (size=113509)
2024-11-10T02:12:08,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741867_1043 (size=113509)
2024-11-10T02:12:08,935 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dfd73d3cab49479db26091ea1d448da3
2024-11-10T02:12:08,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741868_1044 (size=17906)
2024-11-10T02:12:08,941 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/98511745aba6440ab018613d59dfb734 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/98511745aba6440ab018613d59dfb734
2024-11-10T02:12:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/dfd73d3cab49479db26091ea1d448da3 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3
2024-11-10T02:12:08,946 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3, entries=12, sequenceid=248, filesize=17.5 K
2024-11-10T02:12:08,947 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for 05915fec36f2ccb3940bfed44b48c54a in 31ms, sequenceid=248, compaction requested=false
2024-11-10T02:12:08,947 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into 98511745aba6440ab018613d59dfb734(size=110.8 K), total size for store is 147.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:08,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:08,947 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:08,947 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204728888; duration=0sec
2024-11-10T02:12:08,947 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:08,947 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:09,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:09,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:10,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:10,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:10,934 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:12:10,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/19ba98be3d6041ad909021167c259f5b is 1080, key is row0188/info:/1731204728918/Put/seqid=0
2024-11-10T02:12:10,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741869_1045 (size=12518)
2024-11-10T02:12:10,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741869_1045 (size=12518)
2024-11-10T02:12:10,957 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/19ba98be3d6041ad909021167c259f5b
2024-11-10T02:12:10,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/19ba98be3d6041ad909021167c259f5b as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b
2024-11-10T02:12:10,974 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b, entries=7, sequenceid=259, filesize=12.2 K
2024-11-10T02:12:10,976 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 05915fec36f2ccb3940bfed44b48c54a in 43ms, sequenceid=259, compaction requested=true
2024-11-10T02:12:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:10,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:10,976 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-10T02:12:10,976 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-10T02:12:10,978 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4  files of size 164011 starting at candidate #0 after considering 3 permutations with 3 in ratio
2024-11-10T02:12:10,978 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:10,978 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:10,978 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/98511745aba6440ab018613d59dfb734, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=160.2 K
2024-11-10T02:12:10,979 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98511745aba6440ab018613d59dfb734, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731204710503
2024-11-10T02:12:10,980 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f75548e1c44f21985f083ac0076358, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731204728862
2024-11-10T02:12:10,981 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfd73d3cab49479db26091ea1d448da3, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731204728890
2024-11-10T02:12:10,981 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19ba98be3d6041ad909021167c259f5b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731204728918
2024-11-10T02:12:10,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/f5372aec1126456c9457a7c5c75af4c9 is 1080, key is row0195/info:/1731204730935/Put/seqid=0
2024-11-10T02:12:11,004 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#87 average throughput is 45.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:11,005 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/e19c7c5de3cb43bd9f54ae2150ac81a6 is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:11,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741870_1046 (size=19013)
2024-11-10T02:12:11,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741870_1046 (size=19013)
2024-11-10T02:12:11,009 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/f5372aec1126456c9457a7c5c75af4c9
2024-11-10T02:12:11,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/f5372aec1126456c9457a7c5c75af4c9 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9
2024-11-10T02:12:11,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741871_1047 (size=149426)
2024-11-10T02:12:11,019 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741871_1047 (size=149426)
2024-11-10T02:12:11,022 WARN  [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-10T02:12:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:50564 deadline: 1731204741021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134
2024-11-10T02:12:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., hostname=0c4d4d227d60,36531,1731204697134, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., hostname=0c4d4d227d60,36531,1731204697134, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-10T02:12:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., hostname=0c4d4d227d60,36531,1731204697134, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=05915fec36f2ccb3940bfed44b48c54a, server=0c4d4d227d60,36531,1731204697134
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-10T02:12:11,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., hostname=0c4d4d227d60,36531,1731204697134, seqNum=131 because the exception is null or not the one we care about
2024-11-10T02:12:11,027 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9, entries=13, sequenceid=275, filesize=18.6 K
2024-11-10T02:12:11,028 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=16.81 KB/17216 for 05915fec36f2ccb3940bfed44b48c54a in 52ms, sequenceid=275, compaction requested=false
2024-11-10T02:12:11,028 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/e19c7c5de3cb43bd9f54ae2150ac81a6 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e19c7c5de3cb43bd9f54ae2150ac81a6
2024-11-10T02:12:11,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:11,036 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into e19c7c5de3cb43bd9f54ae2150ac81a6(size=145.9 K), total size for store is 164.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:11,036 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:11,036 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=12, startTime=1731204730976; duration=0sec
2024-11-10T02:12:11,036 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:11,036 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:11,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:11,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:12,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:12,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:13,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:13,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:14,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:14,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:15,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:15,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:16,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:16,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:17,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:17,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:18,773 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-10T02:12:18,773 INFO  [master/0c4d4d227d60:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-10T02:12:18,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:18,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:19,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:19,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:20,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:20,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:21,115 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-10T02:12:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/8dbc32bfac3545d7bcc7f6469d37806d is 1080, key is row0208/info:/1731204730978/Put/seqid=0
2024-11-10T02:12:21,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741872_1048 (size=23333)
2024-11-10T02:12:21,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741872_1048 (size=23333)
2024-11-10T02:12:21,134 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/8dbc32bfac3545d7bcc7f6469d37806d
2024-11-10T02:12:21,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/8dbc32bfac3545d7bcc7f6469d37806d as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d
2024-11-10T02:12:21,144 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d, entries=17, sequenceid=296, filesize=22.8 K
2024-11-10T02:12:21,145 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=0 B/0 for 05915fec36f2ccb3940bfed44b48c54a in 30ms, sequenceid=296, compaction requested=true
2024-11-10T02:12:21,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:21,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:21,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:21,145 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:12:21,146 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 191772 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:12:21,146 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:21,146 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:21,146 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e19c7c5de3cb43bd9f54ae2150ac81a6, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=187.3 K
2024-11-10T02:12:21,146 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting e19c7c5de3cb43bd9f54ae2150ac81a6, keycount=133, bloomtype=ROW, size=145.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731204710503
2024-11-10T02:12:21,147 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5372aec1126456c9457a7c5c75af4c9, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731204730935
2024-11-10T02:12:21,147 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dbc32bfac3545d7bcc7f6469d37806d, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731204730978
2024-11-10T02:12:21,158 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#89 average throughput is 55.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:21,159 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/fffb07558be9444a827f9796b709f682 is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:21,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741873_1049 (size=181906)
2024-11-10T02:12:21,162 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741873_1049 (size=181906)
2024-11-10T02:12:21,167 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/fffb07558be9444a827f9796b709f682 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fffb07558be9444a827f9796b709f682
2024-11-10T02:12:21,172 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into fffb07558be9444a827f9796b709f682(size=177.6 K), total size for store is 177.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:21,172 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:21,172 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204741145; duration=0sec
2024-11-10T02:12:21,172 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:21,172 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:21,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:21,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:22,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:22,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:23,124 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-11-10T02:12:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:23,128 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-10T02:12:23,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/db4f3040f7e54ff083a1df03d25f2c62 is 1080, key is row0225/info:/1731204743117/Put/seqid=0
2024-11-10T02:12:23,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741874_1050 (size=12523)
2024-11-10T02:12:23,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741874_1050 (size=12523)
2024-11-10T02:12:23,140 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/db4f3040f7e54ff083a1df03d25f2c62
2024-11-10T02:12:23,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/db4f3040f7e54ff083a1df03d25f2c62 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62
2024-11-10T02:12:23,150 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62, entries=7, sequenceid=307, filesize=12.2 K
2024-11-10T02:12:23,151 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 05915fec36f2ccb3940bfed44b48c54a in 24ms, sequenceid=307, compaction requested=false
2024-11-10T02:12:23,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:23,152 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-10T02:12:23,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/d69eb7b430e84e4c98a0fb44a5094de9 is 1080, key is row0232/info:/1731204743129/Put/seqid=0
2024-11-10T02:12:23,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741875_1051 (size=17918)
2024-11-10T02:12:23,161 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/d69eb7b430e84e4c98a0fb44a5094de9
2024-11-10T02:12:23,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741875_1051 (size=17918)
2024-11-10T02:12:23,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/d69eb7b430e84e4c98a0fb44a5094de9 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9
2024-11-10T02:12:23,171 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9, entries=12, sequenceid=322, filesize=17.5 K
2024-11-10T02:12:23,172 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 05915fec36f2ccb3940bfed44b48c54a in 20ms, sequenceid=322, compaction requested=true
2024-11-10T02:12:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05915fec36f2ccb3940bfed44b48c54a:info, priority=-2147483648, current under compaction store size is 1
2024-11-10T02:12:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:23,172 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-10T02:12:23,173 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 212347 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-10T02:12:23,173 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1541): 05915fec36f2ccb3940bfed44b48c54a/info is initiating minor compaction (all files)
2024-11-10T02:12:23,173 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05915fec36f2ccb3940bfed44b48c54a/info in TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:23,174 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fffb07558be9444a827f9796b709f682, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9] into tmpdir=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp, totalSize=207.4 K
2024-11-10T02:12:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36531 {}] regionserver.HRegion(8855): Flush requested on 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:23,174 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting fffb07558be9444a827f9796b709f682, keycount=163, bloomtype=ROW, size=177.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731204710503
2024-11-10T02:12:23,174 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-10T02:12:23,174 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting db4f3040f7e54ff083a1df03d25f2c62, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731204743117
2024-11-10T02:12:23,175 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] compactions.Compactor(225): Compacting d69eb7b430e84e4c98a0fb44a5094de9, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731204743129
2024-11-10T02:12:23,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/7c8f103bd04c444886cd8d0240a19a3d is 1080, key is row0244/info:/1731204743153/Put/seqid=0
2024-11-10T02:12:23,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741876_1052 (size=17918)
2024-11-10T02:12:23,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741876_1052 (size=17918)
2024-11-10T02:12:23,182 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/7c8f103bd04c444886cd8d0240a19a3d
2024-11-10T02:12:23,187 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05915fec36f2ccb3940bfed44b48c54a#info#compaction#93 average throughput is 62.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-10T02:12:23,188 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/f4c6ed01db164ae8a8dd36a4ef7d630f is 1080, key is row0062/info:/1731204710503/Put/seqid=0
2024-11-10T02:12:23,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741877_1053 (size=202513)
2024-11-10T02:12:23,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741877_1053 (size=202513)
2024-11-10T02:12:23,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/7c8f103bd04c444886cd8d0240a19a3d as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/7c8f103bd04c444886cd8d0240a19a3d
2024-11-10T02:12:23,194 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/7c8f103bd04c444886cd8d0240a19a3d, entries=12, sequenceid=337, filesize=17.5 K
2024-11-10T02:12:23,195 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/f4c6ed01db164ae8a8dd36a4ef7d630f as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f4c6ed01db164ae8a8dd36a4ef7d630f
2024-11-10T02:12:23,195 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=1.05 KB/1076 for 05915fec36f2ccb3940bfed44b48c54a in 21ms, sequenceid=337, compaction requested=false
2024-11-10T02:12:23,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:23,201 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05915fec36f2ccb3940bfed44b48c54a/info of 05915fec36f2ccb3940bfed44b48c54a into f4c6ed01db164ae8a8dd36a4ef7d630f(size=197.8 K), total size for store is 215.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-10T02:12:23,201 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:23,201 INFO  [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a., storeName=05915fec36f2ccb3940bfed44b48c54a/info, priority=13, startTime=1731204743172; duration=0sec
2024-11-10T02:12:23,201 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-10T02:12:23,201 DEBUG [RS:0;0c4d4d227d60:36531-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05915fec36f2ccb3940bfed44b48c54a:info
2024-11-10T02:12:23,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:23,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:24,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:24,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:25,177 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-10T02:12:25,177 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C36531%2C1731204697134.1731204745177
2024-11-10T02:12:25,183 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,183 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,183 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,183 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,183 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,184 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204697761 with entries=317, filesize=309.99 KB; new WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204745177
2024-11-10T02:12:25,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34515:34515),(127.0.0.1/127.0.0.1:33623:33623)]
2024-11-10T02:12:25,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204697761 is not closed yet, will try archiving it next time
2024-11-10T02:12:25,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741833_1009 (size=317435)
2024-11-10T02:12:25,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741833_1009 (size=317435)
2024-11-10T02:12:25,533 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-10T02:12:25,538 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/361912a40c9647ff810583fcc564fc09 is 193, key is TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a./info:regioninfo/1731204713360/Put/seqid=0
2024-11-10T02:12:25,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741879_1055 (size=6223)
2024-11-10T02:12:25,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741879_1055 (size=6223)
2024-11-10T02:12:25,542 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/361912a40c9647ff810583fcc564fc09
2024-11-10T02:12:25,549 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/.tmp/info/361912a40c9647ff810583fcc564fc09 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/info/361912a40c9647ff810583fcc564fc09
2024-11-10T02:12:25,554 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/info/361912a40c9647ff810583fcc564fc09, entries=5, sequenceid=21, filesize=6.1 K
2024-11-10T02:12:25,555 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=21, compaction requested=false
2024-11-10T02:12:25,555 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-10T02:12:25,555 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f6e34468bc8d4c76535cd1cf0979e04f:

2024-11-10T02:12:25,555 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 05915fec36f2ccb3940bfed44b48c54a 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-10T02:12:25,559 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/ad3f658add3343d5b8c501f011423ea7 is 1080, key is row0256/info:/1731204743175/Put/seqid=0
2024-11-10T02:12:25,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741880_1056 (size=6035)
2024-11-10T02:12:25,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741880_1056 (size=6035)
2024-11-10T02:12:25,564 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/ad3f658add3343d5b8c501f011423ea7
2024-11-10T02:12:25,569 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/.tmp/info/ad3f658add3343d5b8c501f011423ea7 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/ad3f658add3343d5b8c501f011423ea7
2024-11-10T02:12:25,575 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/ad3f658add3343d5b8c501f011423ea7, entries=1, sequenceid=342, filesize=5.9 K
2024-11-10T02:12:25,576 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 05915fec36f2ccb3940bfed44b48c54a in 21ms, sequenceid=342, compaction requested=true
2024-11-10T02:12:25,576 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 05915fec36f2ccb3940bfed44b48c54a:

2024-11-10T02:12:25,576 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C36531%2C1731204697134.1731204745576
2024-11-10T02:12:25,585 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,585 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,585 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,585 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,585 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,585 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204745177 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204745576
2024-11-10T02:12:25,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741878_1054 (size=731)
2024-11-10T02:12:25,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741878_1054 (size=731)
2024-11-10T02:12:25,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34515:34515),(127.0.0.1/127.0.0.1:33623:33623)]
2024-11-10T02:12:25,590 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204697761 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs/0c4d4d227d60%2C36531%2C1731204697134.1731204697761
2024-11-10T02:12:25,591 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-10T02:12:25,591 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:12:25,591 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:12:25,591 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:25,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:25,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:25,591 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:12:25,592 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:12:25,592 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1540438130, stopped=false
2024-11-10T02:12:25,592 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,37519,1731204696843
2024-11-10T02:12:25,592 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/WALs/0c4d4d227d60,36531,1731204697134/0c4d4d227d60%2C36531%2C1731204697134.1731204745177 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs/0c4d4d227d60%2C36531%2C1731204697134.1731204745177
2024-11-10T02:12:25,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:25,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:25,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:25,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:25,628 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:12:25,628 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:12:25,628 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:25,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:25,628 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,36531,1731204697134' *****
2024-11-10T02:12:25,628 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:12:25,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:25,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:12:25,629 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:12:25,629 INFO  [regionserver/0c4d4d227d60:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(3091): Received CLOSE for f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(3091): Received CLOSE for 05915fec36f2ccb3940bfed44b48c54a
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,36531,1731204697134
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:36531.
2024-11-10T02:12:25,629 DEBUG [RS:0;0c4d4d227d60:36531 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:25,629 DEBUG [RS:0;0c4d4d227d60:36531 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:25,629 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6e34468bc8d4c76535cd1cf0979e04f, disabling compactions & flushes
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:12:25,629 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:12:25,629 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:12:25,629 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:12:25,630 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f. after waiting 1 ms
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:12:25,630 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-10T02:12:25,630 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f6e34468bc8d4c76535cd1cf0979e04f=TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f., 05915fec36f2ccb3940bfed44b48c54a=TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.}
2024-11-10T02:12:25,630 DEBUG [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1351): Waiting on 05915fec36f2ccb3940bfed44b48c54a, 1588230740, f6e34468bc8d4c76535cd1cf0979e04f
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:12:25,630 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:12:25,630 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:12:25,630 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-bottom] to archive
2024-11-10T02:12:25,631 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:12:25,632 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:12:25,633 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0c4d4d227d60:37519 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-10T02:12:25,633 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-10T02:12:25,634 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-10T02:12:25,635 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:12:25,635 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:12:25,635 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204745630Running coprocessor pre-close hooks at 1731204745630Disabling compacts and flushes for region at 1731204745630Disabling writes for close at 1731204745630Writing region close event to WAL at 1731204745631 (+1 ms)Running coprocessor post-close hooks at 1731204745635 (+4 ms)Closed at 1731204745635
2024-11-10T02:12:25,635 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:12:25,636 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/f6e34468bc8d4c76535cd1cf0979e04f/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-10T02:12:25,636 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:12:25,636 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6e34468bc8d4c76535cd1cf0979e04f:
Waiting for close lock at 1731204745629Running coprocessor pre-close hooks at 1731204745629Disabling compacts and flushes for region at 1731204745629Disabling writes for close at 1731204745630 (+1 ms)Writing region close event to WAL at 1731204745633 (+3 ms)Running coprocessor post-close hooks at 1731204745636 (+3 ms)Closed at 1731204745636
2024-11-10T02:12:25,636 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731204712651.f6e34468bc8d4c76535cd1cf0979e04f.
2024-11-10T02:12:25,637 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05915fec36f2ccb3940bfed44b48c54a, disabling compactions & flushes
2024-11-10T02:12:25,637 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:25,637 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:25,637 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a. after waiting 0 ms
2024-11-10T02:12:25,637 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:25,637 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12->hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/623ea0c0c13a9c5823b3e3663c4f8c12/info/56b4ce752e974b12a2145cf798958f1b-top, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/cf7e1d742dac4e329566c6f78b92354e, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/6b169e5350bd4e229db979a7803ef177, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d8d5688837c54de5ab247fb11be730d2, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/98511745aba6440ab018613d59dfb734, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e19c7c5de3cb43bd9f54ae2150ac81a6, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fffb07558be9444a827f9796b709f682, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9] to archive
2024-11-10T02:12:25,638 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-10T02:12:25,639 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/56b4ce752e974b12a2145cf798958f1b.623ea0c0c13a9c5823b3e3663c4f8c12
2024-11-10T02:12:25,641 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-2e3d91bd147543a9a6197e0e9c61d80a
2024-11-10T02:12:25,642 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/cf7e1d742dac4e329566c6f78b92354e to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/cf7e1d742dac4e329566c6f78b92354e
2024-11-10T02:12:25,643 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/TestLogRolling-testLogRolling=623ea0c0c13a9c5823b3e3663c4f8c12-582bb0c469644ff6a4338b03a0a6e5ae
2024-11-10T02:12:25,644 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/426c176cb35b4826b1c5d8c9ba22f439
2024-11-10T02:12:25,645 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/6b169e5350bd4e229db979a7803ef177 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/6b169e5350bd4e229db979a7803ef177
2024-11-10T02:12:25,646 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/a607995d4359441d8d0ea481a8076615
2024-11-10T02:12:25,648 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e1ff3cb5341340829189ea509ee23bc0
2024-11-10T02:12:25,649 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d8d5688837c54de5ab247fb11be730d2 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d8d5688837c54de5ab247fb11be730d2
2024-11-10T02:12:25,650 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fd4442ac104c4fa3976cff7abf4472ce
2024-11-10T02:12:25,651 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-10T02:12:25,651 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/b2dcc38daf994f31a9a638fd25c3cbfc
2024-11-10T02:12:25,651 INFO  [regionserver/0c4d4d227d60:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-10T02:12:25,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/98511745aba6440ab018613d59dfb734 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/98511745aba6440ab018613d59dfb734
2024-11-10T02:12:25,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dc3fe3914d5343a8a8aff50a473b5496
2024-11-10T02:12:25,654 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/16f75548e1c44f21985f083ac0076358
2024-11-10T02:12:25,656 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/dfd73d3cab49479db26091ea1d448da3
2024-11-10T02:12:25,657 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e19c7c5de3cb43bd9f54ae2150ac81a6 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/e19c7c5de3cb43bd9f54ae2150ac81a6
2024-11-10T02:12:25,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/19ba98be3d6041ad909021167c259f5b
2024-11-10T02:12:25,659 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/f5372aec1126456c9457a7c5c75af4c9
2024-11-10T02:12:25,660 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fffb07558be9444a827f9796b709f682 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/fffb07558be9444a827f9796b709f682
2024-11-10T02:12:25,662 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/8dbc32bfac3545d7bcc7f6469d37806d
2024-11-10T02:12:25,663 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/db4f3040f7e54ff083a1df03d25f2c62
2024-11-10T02:12:25,664 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9 to hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/archive/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/info/d69eb7b430e84e4c98a0fb44a5094de9
2024-11-10T02:12:25,664 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [cf7e1d742dac4e329566c6f78b92354e=42984, 426c176cb35b4826b1c5d8c9ba22f439=12516, 6b169e5350bd4e229db979a7803ef177=64714, a607995d4359441d8d0ea481a8076615=19000, e1ff3cb5341340829189ea509ee23bc0=15750, d8d5688837c54de5ab247fb11be730d2=83215, fd4442ac104c4fa3976cff7abf4472ce=12516, b2dcc38daf994f31a9a638fd25c3cbfc=17906, 98511745aba6440ab018613d59dfb734=113509, dc3fe3914d5343a8a8aff50a473b5496=22238, 16f75548e1c44f21985f083ac0076358=20078, dfd73d3cab49479db26091ea1d448da3=17906, e19c7c5de3cb43bd9f54ae2150ac81a6=149426, 19ba98be3d6041ad909021167c259f5b=12518, f5372aec1126456c9457a7c5c75af4c9=19013, fffb07558be9444a827f9796b709f682=181906, 8dbc32bfac3545d7bcc7f6469d37806d=23333, db4f3040f7e54ff083a1df03d25f2c62=12523, d69eb7b430e84e4c98a0fb44a5094de9=17918]
2024-11-10T02:12:25,668 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/data/default/TestLogRolling-testLogRolling/05915fec36f2ccb3940bfed44b48c54a/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=130
2024-11-10T02:12:25,669 INFO  [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:25,669 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05915fec36f2ccb3940bfed44b48c54a:
Waiting for close lock at 1731204745637Running coprocessor pre-close hooks at 1731204745637Disabling compacts and flushes for region at 1731204745637Disabling writes for close at 1731204745637Writing region close event to WAL at 1731204745665 (+28 ms)Running coprocessor post-close hooks at 1731204745669 (+4 ms)Closed at 1731204745669
2024-11-10T02:12:25,669 DEBUG [RS_CLOSE_REGION-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731204712651.05915fec36f2ccb3940bfed44b48c54a.
2024-11-10T02:12:25,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:25,830 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,36531,1731204697134; all regions closed.
2024-11-10T02:12:25,830 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,831 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,831 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,831 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,831 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:25,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741834_1010 (size=8107)
2024-11-10T02:12:25,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741834_1010 (size=8107)
2024-11-10T02:12:25,836 DEBUG [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs
2024-11-10T02:12:25,836 INFO  [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C36531%2C1731204697134.meta:.meta(num 1731204698117)
2024-11-10T02:12:25,837 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,837 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,837 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,837 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,837 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:25,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741881_1057 (size=780)
2024-11-10T02:12:25,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741881_1057 (size=780)
2024-11-10T02:12:25,842 DEBUG [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/oldWALs
2024-11-10T02:12:25,842 INFO  [RS:0;0c4d4d227d60:36531 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C36531%2C1731204697134:(num 1731204745576)
2024-11-10T02:12:25,842 DEBUG [RS:0;0c4d4d227d60:36531 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:25,842 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:12:25,842 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:12:25,843 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-10T02:12:25,843 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:12:25,843 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:12:25,843 INFO  [RS:0;0c4d4d227d60:36531 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36531
2024-11-10T02:12:25,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,36531,1731204697134
2024-11-10T02:12:25,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:12:25,855 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:12:25,866 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,36531,1731204697134]
2024-11-10T02:12:25,876 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,36531,1731204697134 already deleted, retry=false
2024-11-10T02:12:25,876 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,36531,1731204697134 expired; onlineServers=0
2024-11-10T02:12:25,876 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,37519,1731204696843' *****
2024-11-10T02:12:25,876 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:12:25,877 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:12:25,877 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:12:25,877 DEBUG [M:0;0c4d4d227d60:37519 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:12:25,877 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:12:25,877 DEBUG [M:0;0c4d4d227d60:37519 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:12:25,877 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204697502 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204697502,5,FailOnTimeoutGroup]
2024-11-10T02:12:25,877 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204697502 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204697502,5,FailOnTimeoutGroup]
2024-11-10T02:12:25,877 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:12:25,877 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:12:25,877 DEBUG [M:0;0c4d4d227d60:37519 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:12:25,877 INFO  [M:0;0c4d4d227d60:37519 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:12:25,878 INFO  [M:0;0c4d4d227d60:37519 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:12:25,878 INFO  [M:0;0c4d4d227d60:37519 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:12:25,878 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:12:25,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:12:25,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:25,887 DEBUG [M:0;0c4d4d227d60:37519 {}] zookeeper.ZKUtil(347): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:12:25,887 WARN  [M:0;0c4d4d227d60:37519 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:12:25,887 INFO  [M:0;0c4d4d227d60:37519 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/.lastflushedseqids
2024-11-10T02:12:25,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741882_1058 (size=228)
2024-11-10T02:12:25,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741882_1058 (size=228)
2024-11-10T02:12:25,894 INFO  [M:0;0c4d4d227d60:37519 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:12:25,894 INFO  [M:0;0c4d4d227d60:37519 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:12:25,894 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:12:25,895 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:25,895 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:25,895 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:12:25,895 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:25,895 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.44 KB heapSize=63.39 KB
2024-11-10T02:12:25,917 DEBUG [M:0;0c4d4d227d60:37519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88f835bc18ac47009b8998d00b0775c6 is 82, key is hbase:meta,,1/info:regioninfo/1731204698140/Put/seqid=0
2024-11-10T02:12:25,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741883_1059 (size=5672)
2024-11-10T02:12:25,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741883_1059 (size=5672)
2024-11-10T02:12:25,923 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88f835bc18ac47009b8998d00b0775c6
2024-11-10T02:12:25,950 DEBUG [M:0;0c4d4d227d60:37519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d237d8404f4abfa6eb3a73d561ce5f is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731204698647/Put/seqid=0
2024-11-10T02:12:25,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741884_1060 (size=7091)
2024-11-10T02:12:25,959 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.84 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d237d8404f4abfa6eb3a73d561ce5f
2024-11-10T02:12:25,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741884_1060 (size=7091)
2024-11-10T02:12:25,964 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1d237d8404f4abfa6eb3a73d561ce5f
2024-11-10T02:12:25,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:25,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36531-0x101228346a70001, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:25,966 INFO  [RS:0;0c4d4d227d60:36531 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:12:25,966 INFO  [RS:0;0c4d4d227d60:36531 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,36531,1731204697134; zookeeper connection closed.
2024-11-10T02:12:25,967 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e2d2f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e2d2f3
2024-11-10T02:12:25,967 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:12:25,983 DEBUG [M:0;0c4d4d227d60:37519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d97b31fc4c43cfa5c4af786cd5fbc5 is 69, key is 0c4d4d227d60,36531,1731204697134/rs:state/1731204697603/Put/seqid=0
2024-11-10T02:12:25,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741885_1061 (size=5156)
2024-11-10T02:12:25,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741885_1061 (size=5156)
2024-11-10T02:12:25,988 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d97b31fc4c43cfa5c4af786cd5fbc5
2024-11-10T02:12:26,013 DEBUG [M:0;0c4d4d227d60:37519 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6da467aae69a4eb3b088da8557f1364d is 52, key is load_balancer_on/state:d/1731204698275/Put/seqid=0
2024-11-10T02:12:26,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741886_1062 (size=5056)
2024-11-10T02:12:26,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741886_1062 (size=5056)
2024-11-10T02:12:26,018 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6da467aae69a4eb3b088da8557f1364d
2024-11-10T02:12:26,023 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88f835bc18ac47009b8998d00b0775c6 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/88f835bc18ac47009b8998d00b0775c6
2024-11-10T02:12:26,027 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/88f835bc18ac47009b8998d00b0775c6, entries=8, sequenceid=125, filesize=5.5 K
2024-11-10T02:12:26,028 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1d237d8404f4abfa6eb3a73d561ce5f as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1d237d8404f4abfa6eb3a73d561ce5f
2024-11-10T02:12:26,032 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1d237d8404f4abfa6eb3a73d561ce5f
2024-11-10T02:12:26,033 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1d237d8404f4abfa6eb3a73d561ce5f, entries=13, sequenceid=125, filesize=6.9 K
2024-11-10T02:12:26,034 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d97b31fc4c43cfa5c4af786cd5fbc5 as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97d97b31fc4c43cfa5c4af786cd5fbc5
2024-11-10T02:12:26,039 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97d97b31fc4c43cfa5c4af786cd5fbc5, entries=1, sequenceid=125, filesize=5.0 K
2024-11-10T02:12:26,040 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6da467aae69a4eb3b088da8557f1364d as hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6da467aae69a4eb3b088da8557f1364d
2024-11-10T02:12:26,045 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44877/user/jenkins/test-data/b497f44f-5376-6e4c-218e-c68c6f394c1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6da467aae69a4eb3b088da8557f1364d, entries=1, sequenceid=125, filesize=4.9 K
2024-11-10T02:12:26,046 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=125, compaction requested=false
2024-11-10T02:12:26,047 INFO  [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:26,047 DEBUG [M:0;0c4d4d227d60:37519 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204745894Disabling compacts and flushes for region at 1731204745894Disabling writes for close at 1731204745895 (+1 ms)Obtaining lock to block concurrent updates at 1731204745895Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204745895Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52675, getHeapSize=64848, getOffHeapSize=0, getCellsCount=148 at 1731204745895Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204745896 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204745896Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204745917 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204745917Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204745929 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204745949 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204745949Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204745964 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204745982 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204745982Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204745994 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204746012 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204746012Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2377e5df: reopening flushed file at 1731204746022 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c61bc25: reopening flushed file at 1731204746027 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@147a209b: reopening flushed file at 1731204746033 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75fa7711: reopening flushed file at 1731204746040 (+7 ms)Finished flush of dataSize ~51.44 KB/52675, heapSize ~63.33 KB/64848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 151ms, sequenceid=125, compaction requested=false at 1731204746046 (+6 ms)Writing region close event to WAL at 1731204746047 (+1 ms)Closed at 1731204746047
2024-11-10T02:12:26,048 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:26,048 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:26,048 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:26,048 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:26,048 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:26,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41963 is added to blk_1073741830_1006 (size=61344)
2024-11-10T02:12:26,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36977 is added to blk_1073741830_1006 (size=61344)
2024-11-10T02:12:26,051 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:12:26,051 INFO  [M:0;0c4d4d227d60:37519 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:12:26,051 INFO  [M:0;0c4d4d227d60:37519 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37519
2024-11-10T02:12:26,051 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:12:26,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:26,176 INFO  [M:0;0c4d4d227d60:37519 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:12:26,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37519-0x101228346a70000, quorum=127.0.0.1:58960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:26,180 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@20408ea5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:26,180 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e2ab763{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:26,180 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:26,181 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f4aa33e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:26,181 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47bcda8c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:26,182 WARN  [BP-292189752-172.17.0.3-1731204694004 heartbeating to localhost/127.0.0.1:44877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:12:26,182 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:12:26,182 WARN  [BP-292189752-172.17.0.3-1731204694004 heartbeating to localhost/127.0.0.1:44877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-292189752-172.17.0.3-1731204694004 (Datanode Uuid 9589c007-f254-4173-946d-0b774e725934) service to localhost/127.0.0.1:44877
2024-11-10T02:12:26,182 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:12:26,183 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data3/current/BP-292189752-172.17.0.3-1731204694004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:26,183 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data4/current/BP-292189752-172.17.0.3-1731204694004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:26,183 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:12:26,185 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64358886{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:26,185 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@26a9d62d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:26,185 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:26,185 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@601b78f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:26,186 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23e1642c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:26,187 WARN  [BP-292189752-172.17.0.3-1731204694004 heartbeating to localhost/127.0.0.1:44877 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:12:26,187 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:12:26,187 WARN  [BP-292189752-172.17.0.3-1731204694004 heartbeating to localhost/127.0.0.1:44877 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-292189752-172.17.0.3-1731204694004 (Datanode Uuid a25e57ba-dbec-409c-92eb-94aa44af277f) service to localhost/127.0.0.1:44877
2024-11-10T02:12:26,187 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:12:26,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data1/current/BP-292189752-172.17.0.3-1731204694004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:26,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/cluster_5d053a84-22f9-ac6c-7628-1d9c51341360/data/data2/current/BP-292189752-172.17.0.3-1731204694004 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:26,188 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:12:26,192 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b6783f5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:12:26,193 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44a25975{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:26,193 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:26,193 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f1304aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:26,193 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7193a060{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:26,199 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:12:26,230 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:12:26,239 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 205)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44877
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44877 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44877 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44877
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44877
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44877
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44877
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44877 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 205) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2519 (was 2821)
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=2519
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.log.dir so I do NOT create it in target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ed1a733c-71e8-af6b-084a-06159b80f04b/hadoop.tmp.dir so I do NOT create it in target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297, deleteOnExit=true
2024-11-10T02:12:26,247 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/test.cache.data in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.tmp.dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-10T02:12:26,248 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:12:26,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/nfs.dump.dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/java.io.tmpdir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-10T02:12:26,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-10T02:12:26,260 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:12:26,657 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:12:26,660 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:12:26,664 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:12:26,664 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:12:26,664 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:12:26,665 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:12:26,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35c58925{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:12:26,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b4a20b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:12:26,758 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@133d485{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/java.io.tmpdir/jetty-localhost-43511-hadoop-hdfs-3_4_1-tests_jar-_-any-4639881080548030630/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:12:26,758 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@f33170f{HTTP/1.1, (http/1.1)}{localhost:43511}
2024-11-10T02:12:26,758 INFO  [Time-limited test {}] server.Server(415): Started @304234ms
2024-11-10T02:12:26,769 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-10T02:12:26,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:26,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:27,018 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:12:27,021 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:12:27,022 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:12:27,022 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:12:27,022 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:12:27,022 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1557ab96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:12:27,023 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@271e0d58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:12:27,114 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55931fd2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/java.io.tmpdir/jetty-localhost-36691-hadoop-hdfs-3_4_1-tests_jar-_-any-6389861514748072030/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:27,114 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@554afc70{HTTP/1.1, (http/1.1)}{localhost:36691}
2024-11-10T02:12:27,114 INFO  [Time-limited test {}] server.Server(415): Started @304590ms
2024-11-10T02:12:27,115 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:12:27,140 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-10T02:12:27,143 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-10T02:12:27,143 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-10T02:12:27,144 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-10T02:12:27,144 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-10T02:12:27,144 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32b4a64d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,AVAILABLE}
2024-11-10T02:12:27,144 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4c9b811e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-10T02:12:27,236 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e66231a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/java.io.tmpdir/jetty-localhost-34667-hadoop-hdfs-3_4_1-tests_jar-_-any-12901507153340534481/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:27,236 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@132f2855{HTTP/1.1, (http/1.1)}{localhost:34667}
2024-11-10T02:12:27,236 INFO  [Time-limited test {}] server.Server(415): Started @304712ms
2024-11-10T02:12:27,237 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-10T02:12:27,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:27,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:28,204 WARN  [Thread-2521 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data1/current/BP-1239081256-172.17.0.3-1731204746263/current, will proceed with Du for space computation calculation, 
2024-11-10T02:12:28,204 WARN  [Thread-2522 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data2/current/BP-1239081256-172.17.0.3-1731204746263/current, will proceed with Du for space computation calculation, 
2024-11-10T02:12:28,228 WARN  [Thread-2484 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:12:28,230 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x881f229b024aaeea with lease ID 0x76f46ddfc5438a24: Processing first storage report for DS-03f7cbb3-dbdf-4384-bc39-4e0e108c9079 from datanode DatanodeRegistration(127.0.0.1:38815, datanodeUuid=bf6079c9-51f2-48c9-83fb-7f28ba63f559, infoPort=37295, infoSecurePort=0, ipcPort=40667, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263)
2024-11-10T02:12:28,230 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x881f229b024aaeea with lease ID 0x76f46ddfc5438a24: from storage DS-03f7cbb3-dbdf-4384-bc39-4e0e108c9079 node DatanodeRegistration(127.0.0.1:38815, datanodeUuid=bf6079c9-51f2-48c9-83fb-7f28ba63f559, infoPort=37295, infoSecurePort=0, ipcPort=40667, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:12:28,230 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x881f229b024aaeea with lease ID 0x76f46ddfc5438a24: Processing first storage report for DS-bf7dffaf-9915-4b83-954e-795de71a7afc from datanode DatanodeRegistration(127.0.0.1:38815, datanodeUuid=bf6079c9-51f2-48c9-83fb-7f28ba63f559, infoPort=37295, infoSecurePort=0, ipcPort=40667, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263)
2024-11-10T02:12:28,230 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x881f229b024aaeea with lease ID 0x76f46ddfc5438a24: from storage DS-bf7dffaf-9915-4b83-954e-795de71a7afc node DatanodeRegistration(127.0.0.1:38815, datanodeUuid=bf6079c9-51f2-48c9-83fb-7f28ba63f559, infoPort=37295, infoSecurePort=0, ipcPort=40667, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:12:28,308 WARN  [Thread-2533 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data4/current/BP-1239081256-172.17.0.3-1731204746263/current, will proceed with Du for space computation calculation, 
2024-11-10T02:12:28,308 WARN  [Thread-2532 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data3/current/BP-1239081256-172.17.0.3-1731204746263/current, will proceed with Du for space computation calculation, 
2024-11-10T02:12:28,327 WARN  [Thread-2507 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-10T02:12:28,329 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1914e5613d64fd71 with lease ID 0x76f46ddfc5438a25: Processing first storage report for DS-3c1a13f5-5e63-4b1e-a24d-e73a28f0e82f from datanode DatanodeRegistration(127.0.0.1:44235, datanodeUuid=c9a9a869-78f8-4eda-b885-c3a84cd96fa3, infoPort=33959, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263)
2024-11-10T02:12:28,329 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1914e5613d64fd71 with lease ID 0x76f46ddfc5438a25: from storage DS-3c1a13f5-5e63-4b1e-a24d-e73a28f0e82f node DatanodeRegistration(127.0.0.1:44235, datanodeUuid=c9a9a869-78f8-4eda-b885-c3a84cd96fa3, infoPort=33959, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:12:28,329 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1914e5613d64fd71 with lease ID 0x76f46ddfc5438a25: Processing first storage report for DS-89e3fefc-576a-4215-a7f7-839c183b31e0 from datanode DatanodeRegistration(127.0.0.1:44235, datanodeUuid=c9a9a869-78f8-4eda-b885-c3a84cd96fa3, infoPort=33959, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263)
2024-11-10T02:12:28,330 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1914e5613d64fd71 with lease ID 0x76f46ddfc5438a25: from storage DS-89e3fefc-576a-4215-a7f7-839c183b31e0 node DatanodeRegistration(127.0.0.1:44235, datanodeUuid=c9a9a869-78f8-4eda-b885-c3a84cd96fa3, infoPort=33959, infoSecurePort=0, ipcPort=34885, storageInfo=lv=-57;cid=testClusterID;nsid=619143782;c=1731204746263), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-10T02:12:28,372 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d
2024-11-10T02:12:28,374 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/zookeeper_0, clientPort=57909, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-10T02:12:28,375 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57909
2024-11-10T02:12:28,375 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,376 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:12:28,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741825_1001 (size=7)
2024-11-10T02:12:28,385 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29 with version=8
2024-11-10T02:12:28,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36833/user/jenkins/test-data/129b4a1f-9954-3599-e2cf-e6ac74e037fb/hbase-staging
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:12:28,387 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40077
2024-11-10T02:12:28,388 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40077 connecting to ZooKeeper ensemble=127.0.0.1:57909
2024-11-10T02:12:28,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400770x0, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:12:28,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40077-0x101228410010000 connected
2024-11-10T02:12:28,533 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,534 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,536 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:28,537 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29, hbase.cluster.distributed=false
2024-11-10T02:12:28,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:12:28,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40077
2024-11-10T02:12:28,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40077
2024-11-10T02:12:28,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40077
2024-11-10T02:12:28,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40077
2024-11-10T02:12:28,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40077
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0c4d4d227d60:0 server-side Connection retries=45
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-10T02:12:28,553 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45901
2024-11-10T02:12:28,554 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45901 connecting to ZooKeeper ensemble=127.0.0.1:57909
2024-11-10T02:12:28,555 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,556 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459010x0, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-10T02:12:28,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:459010x0, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:28,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45901-0x101228410010001 connected
2024-11-10T02:12:28,565 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-10T02:12:28,566 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-10T02:12:28,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-10T02:12:28,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-10T02:12:28,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45901
2024-11-10T02:12:28,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45901
2024-11-10T02:12:28,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45901
2024-11-10T02:12:28,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45901
2024-11-10T02:12:28,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45901
2024-11-10T02:12:28,579 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0c4d4d227d60:40077
2024-11-10T02:12:28,579 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:12:28,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:12:28,586 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-10T02:12:28,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,596 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-10T02:12:28,597 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0c4d4d227d60,40077,1731204748386 from backup master directory
2024-11-10T02:12:28,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:12:28,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-10T02:12:28,607 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:12:28,607 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,610 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/hbase.id] with ID: daf949b0-df58-4a31-9af7-c4088e01be68
2024-11-10T02:12:28,610 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/.tmp/hbase.id
2024-11-10T02:12:28,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:12:28,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741826_1002 (size=42)
2024-11-10T02:12:28,616 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/.tmp/hbase.id]:[hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/hbase.id]
2024-11-10T02:12:28,627 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:28,627 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-10T02:12:28,628 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-10T02:12:28,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:12:28,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741827_1003 (size=196)
2024-11-10T02:12:28,644 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-10T02:12:28,645 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-10T02:12:28,645 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:12:28,653 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:12:28,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741828_1004 (size=1189)
2024-11-10T02:12:28,655 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store
2024-11-10T02:12:28,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:12:28,660 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741829_1005 (size=34)
2024-11-10T02:12:28,660 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:12:28,660 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:12:28,660 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:28,660 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:28,660 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:12:28,660 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:28,661 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:28,661 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204748660Disabling compacts and flushes for region at 1731204748660Disabling writes for close at 1731204748660Writing region close event to WAL at 1731204748660Closed at 1731204748660
2024-11-10T02:12:28,661 WARN  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/.initializing
2024-11-10T02:12:28,661 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/WALs/0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,664 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C40077%2C1731204748386, suffix=, logDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/WALs/0c4d4d227d60,40077,1731204748386, archiveDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/oldWALs, maxLogs=10
2024-11-10T02:12:28,664 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C40077%2C1731204748386.1731204748664
2024-11-10T02:12:28,668 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/WALs/0c4d4d227d60,40077,1731204748386/0c4d4d227d60%2C40077%2C1731204748386.1731204748664
2024-11-10T02:12:28,673 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37295:37295),(127.0.0.1/127.0.0.1:33959:33959)]
2024-11-10T02:12:28,677 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:12:28,677 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:12:28,677 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,677 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,678 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:12:28,679 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-10T02:12:28,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,680 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:28,680 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:12:28,681 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-10T02:12:28,681 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,681 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:12:28,681 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:12:28,682 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-10T02:12:28,682 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,683 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:12:28,683 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-10T02:12:28,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-10T02:12:28,684 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-10T02:12:28,685 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,685 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,686 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,687 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,687 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,687 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-10T02:12:28,688 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-10T02:12:28,690 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:12:28,690 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=816030, jitterRate=0.037636950612068176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-10T02:12:28,691 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731204748677Initializing all the Stores at 1731204748678 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204748678Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204748678Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204748678Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204748678Cleaning up temporary data from old regions at 1731204748687 (+9 ms)Region opened successfully at 1731204748691 (+4 ms)
2024-11-10T02:12:28,692 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-10T02:12:28,695 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5378371c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:12:28,695 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-10T02:12:28,696 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-10T02:12:28,696 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-10T02:12:28,696 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-10T02:12:28,696 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-10T02:12:28,697 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-10T02:12:28,697 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-10T02:12:28,699 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-10T02:12:28,700 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-10T02:12:28,712 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-10T02:12:28,712 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-10T02:12:28,713 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-10T02:12:28,722 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-10T02:12:28,723 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-10T02:12:28,724 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-10T02:12:28,733 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-10T02:12:28,734 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-10T02:12:28,743 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-10T02:12:28,746 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-10T02:12:28,754 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-10T02:12:28,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:28,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:28,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,765 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0c4d4d227d60,40077,1731204748386, sessionid=0x101228410010000, setting cluster-up flag (Was=false)
2024-11-10T02:12:28,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:28,817 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-10T02:12:28,818 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:28,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:28,870 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-10T02:12:28,870 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:28,871 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-10T02:12:28,873 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-10T02:12:28,873 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-10T02:12:28,873 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-10T02:12:28,873 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0c4d4d227d60,40077,1731204748386
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-10T02:12:28,874 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=5, maxPoolSize=5
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0c4d4d227d60:0, corePoolSize=10, maxPoolSize=10
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:12:28,875 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:28,875 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731204778875
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:28,876 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-10T02:12:28,876 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-10T02:12:28,876 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-10T02:12:28,877 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204748877,5,FailOnTimeoutGroup]
2024-11-10T02:12:28,877 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204748877,5,FailOnTimeoutGroup]
2024-11-10T02:12:28,877 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:28,877 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-10T02:12:28,877 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:28,877 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:28,877 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,877 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-10T02:12:28,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:12:28,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741831_1007 (size=1321)
2024-11-10T02:12:28,883 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-10T02:12:28,883 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29
2024-11-10T02:12:28,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:12:28,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741832_1008 (size=32)
2024-11-10T02:12:28,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:12:28,890 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:12:28,891 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:12:28,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,891 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:28,891 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:12:28,892 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:12:28,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:28,892 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:12:28,893 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:12:28,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,893 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:28,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:12:28,894 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:12:28,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:28,895 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:28,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:12:28,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740
2024-11-10T02:12:28,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740
2024-11-10T02:12:28,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:12:28,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:12:28,897 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:12:28,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:12:28,899 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-10T02:12:28,900 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713958, jitterRate=-0.09215672314167023}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:12:28,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731204748889Initializing all the Stores at 1731204748890 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204748890Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204748890Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204748890Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204748890Cleaning up temporary data from old regions at 1731204748896 (+6 ms)Region opened successfully at 1731204748900 (+4 ms)
2024-11-10T02:12:28,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:12:28,900 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:12:28,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:12:28,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:12:28,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:12:28,901 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:12:28,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204748900Disabling compacts and flushes for region at 1731204748900Disabling writes for close at 1731204748900Writing region close event to WAL at 1731204748901 (+1 ms)Closed at 1731204748901
2024-11-10T02:12:28,902 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:12:28,902 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-10T02:12:28,902 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-10T02:12:28,903 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:12:28,904 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-10T02:12:28,970 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(746): ClusterId : daf949b0-df58-4a31-9af7-c4088e01be68
2024-11-10T02:12:28,970 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-10T02:12:28,976 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-10T02:12:28,976 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-10T02:12:28,986 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-10T02:12:28,987 DEBUG [RS:0;0c4d4d227d60:45901 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3be4b9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0c4d4d227d60/172.17.0.3:0
2024-11-10T02:12:28,997 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0c4d4d227d60:45901
2024-11-10T02:12:28,997 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-10T02:12:28,997 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-10T02:12:28,997 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-10T02:12:28,998 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(2659): reportForDuty to master=0c4d4d227d60,40077,1731204748386 with port=45901, startcode=1731204748552
2024-11-10T02:12:28,998 DEBUG [RS:0;0c4d4d227d60:45901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-10T02:12:28,999 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55847, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-10T02:12:29,000 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40077 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,000 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40077 {}] master.ServerManager(517): Registering regionserver=0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,001 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29
2024-11-10T02:12:29,001 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38579
2024-11-10T02:12:29,001 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-10T02:12:29,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:12:29,007 DEBUG [RS:0;0c4d4d227d60:45901 {}] zookeeper.ZKUtil(111): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,007 WARN  [RS:0;0c4d4d227d60:45901 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-10T02:12:29,007 INFO  [RS:0;0c4d4d227d60:45901 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:12:29,007 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,007 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0c4d4d227d60,45901,1731204748552]
2024-11-10T02:12:29,010 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-10T02:12:29,012 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-10T02:12:29,012 INFO  [RS:0;0c4d4d227d60:45901 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-10T02:12:29,012 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,012 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-10T02:12:29,013 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-10T02:12:29,013 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0c4d4d227d60:0, corePoolSize=2, maxPoolSize=2
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0c4d4d227d60:0, corePoolSize=1, maxPoolSize=1
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:12:29,013 DEBUG [RS:0;0c4d4d227d60:45901 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0c4d4d227d60:0, corePoolSize=3, maxPoolSize=3
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,014 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,45901,1731204748552-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:12:29,026 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-10T02:12:29,026 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,45901,1731204748552-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,027 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,027 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.Replication(171): 0c4d4d227d60,45901,1731204748552 started
2024-11-10T02:12:29,039 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,039 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1482): Serving as 0c4d4d227d60,45901,1731204748552, RpcServer on 0c4d4d227d60/172.17.0.3:45901, sessionid=0x101228410010001
2024-11-10T02:12:29,039 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-10T02:12:29,039 DEBUG [RS:0;0c4d4d227d60:45901 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,039 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,45901,1731204748552'
2024-11-10T02:12:29,039 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0c4d4d227d60,45901,1731204748552'
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-10T02:12:29,040 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-10T02:12:29,041 DEBUG [RS:0;0c4d4d227d60:45901 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-10T02:12:29,041 INFO  [RS:0;0c4d4d227d60:45901 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-10T02:12:29,041 INFO  [RS:0;0c4d4d227d60:45901 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-10T02:12:29,054 WARN  [0c4d4d227d60:40077 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-10T02:12:29,143 INFO  [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C45901%2C1731204748552, suffix=, logDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/0c4d4d227d60,45901,1731204748552, archiveDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs, maxLogs=32
2024-11-10T02:12:29,143 INFO  [RS:0;0c4d4d227d60:45901 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C45901%2C1731204748552.1731204749143
2024-11-10T02:12:29,150 INFO  [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/0c4d4d227d60,45901,1731204748552/0c4d4d227d60%2C45901%2C1731204748552.1731204749143
2024-11-10T02:12:29,155 DEBUG [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37295:37295),(127.0.0.1/127.0.0.1:33959:33959)]
2024-11-10T02:12:29,304 DEBUG [0c4d4d227d60:40077 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-10T02:12:29,305 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,306 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,45901,1731204748552, state=OPENING
2024-11-10T02:12:29,312 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-10T02:12:29,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:29,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:29,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:12:29,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:12:29,323 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-10T02:12:29,323 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,45901,1731204748552}]
2024-11-10T02:12:29,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:12:29,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-10T02:12:29,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-10T02:12:29,476 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-10T02:12:29,477 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53621, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-10T02:12:29,480 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-10T02:12:29,480 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:12:29,482 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0c4d4d227d60%2C45901%2C1731204748552.meta, suffix=.meta, logDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/0c4d4d227d60,45901,1731204748552, archiveDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs, maxLogs=32
2024-11-10T02:12:29,482 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0c4d4d227d60%2C45901%2C1731204748552.meta.1731204749482.meta
2024-11-10T02:12:29,488 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/0c4d4d227d60,45901,1731204748552/0c4d4d227d60%2C45901%2C1731204748552.meta.1731204749482.meta
2024-11-10T02:12:29,492 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33959:33959),(127.0.0.1/127.0.0.1:37295:37295)]
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-10T02:12:29,499 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-10T02:12:29,499 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-10T02:12:29,501 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-10T02:12:29,501 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-10T02:12:29,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:29,502 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:29,502 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-10T02:12:29,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-10T02:12:29,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:29,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:29,503 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-10T02:12:29,503 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-10T02:12:29,503 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:29,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:29,504 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-10T02:12:29,504 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-10T02:12:29,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-10T02:12:29,505 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-10T02:12:29,505 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-10T02:12:29,505 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740
2024-11-10T02:12:29,506 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740
2024-11-10T02:12:29,507 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-10T02:12:29,507 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-10T02:12:29,508 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-10T02:12:29,509 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-10T02:12:29,510 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=853102, jitterRate=0.08477610349655151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-10T02:12:29,510 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-10T02:12:29,510 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731204749499Writing region info on filesystem at 1731204749499Initializing all the Stores at 1731204749500 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204749500Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204749500Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731204749500Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731204749501 (+1 ms)Cleaning up temporary data from old regions at 1731204749507 (+6 ms)Running coprocessor post-open hooks at 1731204749510 (+3 ms)Region opened successfully at 1731204749510
2024-11-10T02:12:29,511 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731204749475
2024-11-10T02:12:29,513 DEBUG [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-10T02:12:29,513 INFO  [RS_OPEN_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-10T02:12:29,513 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,514 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0c4d4d227d60,45901,1731204748552, state=OPEN
2024-11-10T02:12:29,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:12:29,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-10T02:12:29,585 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:12:29,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-10T02:12:29,587 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-10T02:12:29,587 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0c4d4d227d60,45901,1731204748552 in 262 msec
2024-11-10T02:12:29,589 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-10T02:12:29,589 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 685 msec
2024-11-10T02:12:29,590 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-10T02:12:29,590 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-10T02:12:29,591 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:12:29,591 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,45901,1731204748552, seqNum=-1]
2024-11-10T02:12:29,591 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:12:29,592 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53591, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:12:29,597 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 723 msec
2024-11-10T02:12:29,597 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731204749597, completionTime=-1
2024-11-10T02:12:29,597 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-10T02:12:29,597 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731204809599
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731204869599
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0c4d4d227d60:40077, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,599 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,601 DEBUG [master/0c4d4d227d60:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-10T02:12:29,602 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.995sec
2024-11-10T02:12:29,602 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-10T02:12:29,602 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-10T02:12:29,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-10T02:12:29,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-10T02:12:29,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-10T02:12:29,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-10T02:12:29,603 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-10T02:12:29,605 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-10T02:12:29,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-10T02:12:29,605 INFO  [master/0c4d4d227d60:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0c4d4d227d60,40077,1731204748386-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-10T02:12:29,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ec9926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:12:29,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0c4d4d227d60,40077,-1 for getting cluster id
2024-11-10T02:12:29,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-10T02:12:29,672 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'daf949b0-df58-4a31-9af7-c4088e01be68'
2024-11-10T02:12:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-10T02:12:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "daf949b0-df58-4a31-9af7-c4088e01be68"

2024-11-10T02:12:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0d0260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:12:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0c4d4d227d60,40077,-1]
2024-11-10T02:12:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-10T02:12:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,674 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-10T02:12:29,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6310a48b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-10T02:12:29,675 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-10T02:12:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0c4d4d227d60,45901,1731204748552, seqNum=-1]
2024-11-10T02:12:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-10T02:12:29,676 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-10T02:12:29,678 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:29,678 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-10T02:12:29,680 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-10T02:12:29,680 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-10T02:12:29,682 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1, archiveDir=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs, maxLogs=32
2024-11-10T02:12:29,682 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731204749682
2024-11-10T02:12:29,687 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1/test.com%2C8080%2C1.1731204749682
2024-11-10T02:12:29,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37295:37295),(127.0.0.1/127.0.0.1:33959:33959)]
2024-11-10T02:12:29,689 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731204749689
2024-11-10T02:12:29,693 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,693 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,693 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,693 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,693 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,693 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1/test.com%2C8080%2C1.1731204749682 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1/test.com%2C8080%2C1.1731204749689
2024-11-10T02:12:29,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33959:33959),(127.0.0.1/127.0.0.1:37295:37295)]
2024-11-10T02:12:29,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1/test.com%2C8080%2C1.1731204749682 is not closed yet, will try archiving it next time
2024-11-10T02:12:29,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741835_1011 (size=93)
2024-11-10T02:12:29,695 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,695 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741835_1011 (size=93)
2024-11-10T02:12:29,695 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,695 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,695 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,695 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,695 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/WALs/test.com,8080,1/test.com%2C8080%2C1.1731204749682 to hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs/test.com%2C8080%2C1.1731204749682
2024-11-10T02:12:29,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741836_1012 (size=93)
2024-11-10T02:12:29,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741836_1012 (size=93)
2024-11-10T02:12:29,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs
2024-11-10T02:12:29,698 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731204749689)
2024-11-10T02:12:29,698 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-10T02:12:29,698 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:12:29,698 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,698 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-10T02:12:29,698 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-10T02:12:29,699 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1378895791, stopped=false
2024-11-10T02:12:29,699 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0c4d4d227d60,40077,1731204748386
2024-11-10T02:12:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-10T02:12:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:29,712 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:12:29,712 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-10T02:12:29,712 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:29,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:29,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-10T02:12:29,712 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0c4d4d227d60,45901,1731204748552' *****
2024-11-10T02:12:29,712 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-10T02:12:29,713 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(959): stopping server 0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0c4d4d227d60:45901.
2024-11-10T02:12:29,713 DEBUG [RS:0;0c4d4d227d60:45901 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-10T02:12:29,713 DEBUG [RS:0;0c4d4d227d60:45901 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-10T02:12:29,713 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-10T02:12:29,713 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-10T02:12:29,713 DEBUG [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-10T02:12:29,713 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-10T02:12:29,713 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-10T02:12:29,713 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-10T02:12:29,713 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-10T02:12:29,713 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-10T02:12:29,714 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-10T02:12:29,727 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/.tmp/ns/396fa7b70fb74a888eb6d49bc5e6578c is 43, key is default/ns:d/1731204749593/Put/seqid=0
2024-11-10T02:12:29,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741837_1013 (size=5153)
2024-11-10T02:12:29,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741837_1013 (size=5153)
2024-11-10T02:12:29,731 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/.tmp/ns/396fa7b70fb74a888eb6d49bc5e6578c
2024-11-10T02:12:29,736 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/.tmp/ns/396fa7b70fb74a888eb6d49bc5e6578c as hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/ns/396fa7b70fb74a888eb6d49bc5e6578c
2024-11-10T02:12:29,741 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/ns/396fa7b70fb74a888eb6d49bc5e6578c, entries=2, sequenceid=6, filesize=5.0 K
2024-11-10T02:12:29,742 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false
2024-11-10T02:12:29,745 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-10T02:12:29,746 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-10T02:12:29,746 INFO  [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-10T02:12:29,746 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731204749713Running coprocessor pre-close hooks at 1731204749713Disabling compacts and flushes for region at 1731204749713Disabling writes for close at 1731204749713Obtaining lock to block concurrent updates at 1731204749714 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731204749714Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731204749714Flushing stores of hbase:meta,,1.1588230740 at 1731204749714Flushing 1588230740/ns: creating writer at 1731204749714Flushing 1588230740/ns: appending metadata at 1731204749726 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1731204749726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46885d3e: reopening flushed file at 1731204749735 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false at 1731204749742 (+7 ms)Writing region close event to WAL at 1731204749742Running coprocessor post-close hooks at 1731204749746 (+4 ms)Closed at 1731204749746
2024-11-10T02:12:29,746 DEBUG [RS_CLOSE_META-regionserver/0c4d4d227d60:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-10T02:12:29,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,46049,1731204555690/0c4d4d227d60%2C46049%2C1731204555690.1731204555996
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:29,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37393/user/jenkins/test-data/c6a3008d-3222-cb46-ccb8-566e90f032a7/WALs/0c4d4d227d60,41989,1731204554095/0c4d4d227d60%2C41989%2C1731204554095.meta.1731204555357.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-10T02:12:29,913 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(976): stopping server 0c4d4d227d60,45901,1731204748552; all regions closed.
2024-11-10T02:12:29,914 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,914 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,914 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,914 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,914 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741834_1010 (size=1152)
2024-11-10T02:12:29,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741834_1010 (size=1152)
2024-11-10T02:12:29,918 DEBUG [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs
2024-11-10T02:12:29,918 INFO  [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C45901%2C1731204748552.meta:.meta(num 1731204749482)
2024-11-10T02:12:29,919 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,919 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,919 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,919 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,919 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:29,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741833_1009 (size=93)
2024-11-10T02:12:29,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741833_1009 (size=93)
2024-11-10T02:12:29,923 DEBUG [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/oldWALs
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0c4d4d227d60%2C45901%2C1731204748552:(num 1731204749143)
2024-11-10T02:12:29,923 DEBUG [RS:0;0c4d4d227d60:45901 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.LeaseManager(133): Closed leases
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.ChoreService(370): Chore service for: regionserver/0c4d4d227d60:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:12:29,923 INFO  [regionserver/0c4d4d227d60:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:12:29,923 INFO  [RS:0;0c4d4d227d60:45901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45901
2024-11-10T02:12:29,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0c4d4d227d60,45901,1731204748552
2024-11-10T02:12:29,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-10T02:12:29,933 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:12:29,943 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0c4d4d227d60,45901,1731204748552]
2024-11-10T02:12:29,954 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0c4d4d227d60,45901,1731204748552 already deleted, retry=false
2024-11-10T02:12:29,954 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0c4d4d227d60,45901,1731204748552 expired; onlineServers=0
2024-11-10T02:12:29,954 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0c4d4d227d60,40077,1731204748386' *****
2024-11-10T02:12:29,954 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-10T02:12:29,954 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-10T02:12:29,954 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-10T02:12:29,954 DEBUG [M:0;0c4d4d227d60:40077 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-10T02:12:29,954 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-10T02:12:29,954 DEBUG [M:0;0c4d4d227d60:40077 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-10T02:12:29,954 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204748877 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.small.0-1731204748877,5,FailOnTimeoutGroup]
2024-11-10T02:12:29,954 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204748877 {}] cleaner.HFileCleaner(306): Exit Thread[master/0c4d4d227d60:0:becomeActiveMaster-HFileCleaner.large.0-1731204748877,5,FailOnTimeoutGroup]
2024-11-10T02:12:29,954 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.ChoreService(370): Chore service for: master/0c4d4d227d60:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-10T02:12:29,955 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-10T02:12:29,955 DEBUG [M:0;0c4d4d227d60:40077 {}] master.HMaster(1795): Stopping service threads
2024-11-10T02:12:29,955 INFO  [M:0;0c4d4d227d60:40077 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-10T02:12:29,955 INFO  [M:0;0c4d4d227d60:40077 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-10T02:12:29,955 INFO  [M:0;0c4d4d227d60:40077 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-10T02:12:29,955 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-10T02:12:29,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-10T02:12:29,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-10T02:12:29,964 DEBUG [M:0;0c4d4d227d60:40077 {}] zookeeper.ZKUtil(347): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-10T02:12:29,964 WARN  [M:0;0c4d4d227d60:40077 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-10T02:12:29,965 INFO  [M:0;0c4d4d227d60:40077 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/.lastflushedseqids
2024-11-10T02:12:29,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741838_1014 (size=99)
2024-11-10T02:12:29,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741838_1014 (size=99)
2024-11-10T02:12:29,970 INFO  [M:0;0c4d4d227d60:40077 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-10T02:12:29,971 INFO  [M:0;0c4d4d227d60:40077 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-10T02:12:29,971 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-10T02:12:29,971 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:29,971 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:29,971 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-10T02:12:29,971 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:29,971 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-10T02:12:29,985 DEBUG [M:0;0c4d4d227d60:40077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f141a474e8704a0a89cf9f42b3227f60 is 82, key is hbase:meta,,1/info:regioninfo/1731204749513/Put/seqid=0
2024-11-10T02:12:29,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741839_1015 (size=5672)
2024-11-10T02:12:29,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741839_1015 (size=5672)
2024-11-10T02:12:29,990 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f141a474e8704a0a89cf9f42b3227f60
2024-11-10T02:12:30,006 DEBUG [M:0;0c4d4d227d60:40077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc99d3f646af4eb283989d1db46df20c is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731204749596/Put/seqid=0
2024-11-10T02:12:30,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741840_1016 (size=5275)
2024-11-10T02:12:30,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741840_1016 (size=5275)
2024-11-10T02:12:30,011 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc99d3f646af4eb283989d1db46df20c
2024-11-10T02:12:30,033 DEBUG [M:0;0c4d4d227d60:40077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b0629cb1174805bc4152166b05d55e is 69, key is 0c4d4d227d60,45901,1731204748552/rs:state/1731204749000/Put/seqid=0
2024-11-10T02:12:30,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741841_1017 (size=5156)
2024-11-10T02:12:30,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741841_1017 (size=5156)
2024-11-10T02:12:30,038 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b0629cb1174805bc4152166b05d55e
2024-11-10T02:12:30,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:30,043 INFO  [RS:0;0c4d4d227d60:45901 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:12:30,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45901-0x101228410010001, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:30,044 INFO  [RS:0;0c4d4d227d60:45901 {}] regionserver.HRegionServer(1031): Exiting; stopping=0c4d4d227d60,45901,1731204748552; zookeeper connection closed.
2024-11-10T02:12:30,044 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39e2b786 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39e2b786
2024-11-10T02:12:30,044 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-10T02:12:30,051 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=5, created chunk count=9, reused chunk count=80, reuseRatio=89.89%
2024-11-10T02:12:30,051 DEBUG [master/0c4d4d227d60:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-11-10T02:12:30,060 DEBUG [M:0;0c4d4d227d60:40077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/481b959febdf4da29e657ac70b6a8693 is 52, key is load_balancer_on/state:d/1731204749679/Put/seqid=0
2024-11-10T02:12:30,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741842_1018 (size=5056)
2024-11-10T02:12:30,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741842_1018 (size=5056)
2024-11-10T02:12:30,065 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/481b959febdf4da29e657ac70b6a8693
2024-11-10T02:12:30,070 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f141a474e8704a0a89cf9f42b3227f60 as hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f141a474e8704a0a89cf9f42b3227f60
2024-11-10T02:12:30,074 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f141a474e8704a0a89cf9f42b3227f60, entries=8, sequenceid=29, filesize=5.5 K
2024-11-10T02:12:30,074 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc99d3f646af4eb283989d1db46df20c as hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc99d3f646af4eb283989d1db46df20c
2024-11-10T02:12:30,079 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc99d3f646af4eb283989d1db46df20c, entries=3, sequenceid=29, filesize=5.2 K
2024-11-10T02:12:30,080 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/23b0629cb1174805bc4152166b05d55e as hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23b0629cb1174805bc4152166b05d55e
2024-11-10T02:12:30,084 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/23b0629cb1174805bc4152166b05d55e, entries=1, sequenceid=29, filesize=5.0 K
2024-11-10T02:12:30,085 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/481b959febdf4da29e657ac70b6a8693 as hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/481b959febdf4da29e657ac70b6a8693
2024-11-10T02:12:30,089 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38579/user/jenkins/test-data/7d34db20-b846-8ede-a6f6-b3da55a56c29/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/481b959febdf4da29e657ac70b6a8693, entries=1, sequenceid=29, filesize=4.9 K
2024-11-10T02:12:30,090 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=29, compaction requested=false
2024-11-10T02:12:30,091 INFO  [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-10T02:12:30,091 DEBUG [M:0;0c4d4d227d60:40077 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731204749971Disabling compacts and flushes for region at 1731204749971Disabling writes for close at 1731204749971Obtaining lock to block concurrent updates at 1731204749971Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731204749971Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731204749971Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731204749972 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731204749972Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731204749984 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731204749985 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731204749993 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731204750006 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731204750006Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731204750015 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731204750032 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731204750033 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731204750041 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731204750059 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731204750059Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42f90214: reopening flushed file at 1731204750069 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689d7d09: reopening flushed file at 1731204750074 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b325aa5: reopening flushed file at 1731204750079 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@437a2461: reopening flushed file at 1731204750084 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=29, compaction requested=false at 1731204750090 (+6 ms)Writing region close event to WAL at 1731204750091 (+1 ms)Closed at 1731204750091
2024-11-10T02:12:30,091 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:30,091 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:30,091 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:30,091 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:30,092 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-10T02:12:30,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44235 is added to blk_1073741830_1006 (size=10311)
2024-11-10T02:12:30,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38815 is added to blk_1073741830_1006 (size=10311)
2024-11-10T02:12:30,094 INFO  [M:0;0c4d4d227d60:40077 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-10T02:12:30,094 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-10T02:12:30,094 INFO  [M:0;0c4d4d227d60:40077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40077
2024-11-10T02:12:30,094 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-10T02:12:30,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:30,201 INFO  [M:0;0c4d4d227d60:40077 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-10T02:12:30,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40077-0x101228410010000, quorum=127.0.0.1:57909, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-10T02:12:30,204 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e66231a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:30,204 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@132f2855{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:30,204 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:30,205 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4c9b811e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:30,205 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32b4a64d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:30,205 WARN  [BP-1239081256-172.17.0.3-1731204746263 heartbeating to localhost/127.0.0.1:38579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:12:30,205 WARN  [BP-1239081256-172.17.0.3-1731204746263 heartbeating to localhost/127.0.0.1:38579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1239081256-172.17.0.3-1731204746263 (Datanode Uuid c9a9a869-78f8-4eda-b885-c3a84cd96fa3) service to localhost/127.0.0.1:38579
2024-11-10T02:12:30,205 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:12:30,206 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:12:30,206 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data3/current/BP-1239081256-172.17.0.3-1731204746263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:30,206 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data4/current/BP-1239081256-172.17.0.3-1731204746263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:30,206 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:12:30,208 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55931fd2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-10T02:12:30,209 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@554afc70{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:30,209 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:30,209 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@271e0d58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:30,209 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1557ab96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:30,210 WARN  [BP-1239081256-172.17.0.3-1731204746263 heartbeating to localhost/127.0.0.1:38579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-10T02:12:30,210 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-10T02:12:30,210 WARN  [BP-1239081256-172.17.0.3-1731204746263 heartbeating to localhost/127.0.0.1:38579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1239081256-172.17.0.3-1731204746263 (Datanode Uuid bf6079c9-51f2-48c9-83fb-7f28ba63f559) service to localhost/127.0.0.1:38579
2024-11-10T02:12:30,210 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-10T02:12:30,210 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data1/current/BP-1239081256-172.17.0.3-1731204746263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:30,210 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/cluster_1090a33d-7277-766a-e21e-6a0f293d6297/data/data2/current/BP-1239081256-172.17.0.3-1731204746263 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-10T02:12:30,211 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-10T02:12:30,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@133d485{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-10T02:12:30,215 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@f33170f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-10T02:12:30,215 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-10T02:12:30,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b4a20b3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-10T02:12:30,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35c58925{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c46470c-a9c1-9f29-6b64-4b307800c39d/hadoop.log.dir/,STOPPED}
2024-11-10T02:12:30,220 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-10T02:12:30,233 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-10T02:12:30,244 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 230)
Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38579
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38579
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38579 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38579 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:38579
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38579
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38579
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38579 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0c4d4d227d60:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2521 (was 2519) - AvailableMemoryMB LEAK? -